{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cell_id": "9117b11ef6ce47a189467e17c80a4a00",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 31
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 1678,
    "execution_start": 1672344851887,
    "source_hash": "d5d7c02b",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 16)\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.tree import DecisionTreeClassifier as DTC\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 创建随机数据集\n",
    "X, y = make_classification(\n",
    "    n_samples=1000, # 数据集大小\n",
    "    n_features=16, # 特征数，即数据维度\n",
    "    n_informative=5, # 有效特征个数\n",
    "    n_redundant=2, # 冗余特征个数，为有效特征的随机线性组合\n",
    "    n_classes=2, # 类别数\n",
    "    flip_y=0.1, # 类别随机的样本个数，该值越大，分类越困难\n",
    "    random_state=0 # 随机种子\n",
    ")\n",
    "\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cell_id": "09f59f60caa4413199481eac15c08d10",
    "collapsed": true,
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 43
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 3,
    "execution_start": 1672344859132,
    "id": "D48D6EFE522D416CA560F56280404245",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "source_hash": "b94c010d",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class RandomForest():\n",
    "\n",
    "    def __init__(self, n_trees=10, max_features='sqrt'):\n",
    "        # max_features是DTC的参数，表示结点分裂时随机采样的特征个数\n",
    "        # sqrt代表取全部特征的平方根，None代表取全部特征，log2代表取全部特征的对数\n",
    "        self.n_trees = n_trees\n",
    "        self.oob_score = 0\n",
    "        self.trees = [DTC(max_features=max_features) for _ in range(n_trees)]\n",
    "\n",
    "    # 用X和y训练模型\n",
    "    def fit(self, X, y):\n",
    "        n_samples, n_features = X.shape\n",
    "        self.n_classes = np.unique(y).shape[0]   \n",
    "        # 集成模型的预测，累加单个模型预测的分类概率，再取较大值作为最终分类\n",
    "        ensemble = np.zeros((n_samples, self.n_classes))\n",
    "            \n",
    "        for tree in self.trees:\n",
    "            # 自举采样，该采样允许重复\n",
    "            idx = np.random.randint(0, n_samples, n_samples)\n",
    "            # 没有被采到的样本\n",
    "            unsampled_mask = np.bincount(idx, minlength=n_samples) == 0\n",
    "            unsampled_idx = np.arange(n_samples)[unsampled_mask]\n",
    "            # 训练当前决策树\n",
    "            tree.fit(X[idx], y[idx])\n",
    "            # 累加决策树对OOB样本的预测\n",
    "            ensemble[unsampled_idx] += tree.predict_proba(X[unsampled_idx])\n",
    "        # 计算OOB分数，由于是分类任务，我们用正确率来衡量\n",
    "        self.oob_score = np.mean(y == np.argmax(ensemble, axis=1))\n",
    "    \n",
    "    # 预测类别\n",
    "    def predict(self, X):\n",
    "        proba = self.predict_proba(X)\n",
    "        return np.argmax(proba, axis=1)\n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "        # 取所有决策树预测概率的平均\n",
    "        ensemble = np.mean([tree.predict_proba(X) for tree in self.trees], axis=0)\n",
    "        return ensemble\n",
    "    \n",
    "    # 计算正确率\n",
    "    def score(self, X, y):\n",
    "        return np.mean(y == self.predict(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cell_id": "9ade2debbd28433c9195a949f6ef6e7a",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 55
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 19971,
    "execution_start": 1672344862088,
    "id": "B3082D1F8B214B48B0374D31B9DF813F",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "source_hash": "b86f3a",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 20/20 [00:08<00:00,  2.47it/s, n_tree=96, train_score=1, oob_score=0.888]\n",
      "100%|███████████████████████████████████████| 20/20 [00:02<00:00,  9.23it/s, n_tree=96, train_score=1, oob_score=0.897]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEKCAYAAADq59mMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FVX6wPHvSQhJ6IHQa1SkpRFC7yBlaaKCCD8URAFB\ndG2IrgXEgoqubbHgCii6CGJZXBEEBBGpQZr0QIJ0QwkQ0pP398dJbyQhNzeB9/M88+SeOVPem5vM\ne2fmzDlGRFBKKaUKwsXZASillCp9NHkopZQqME0eSimlCkyTh1JKqQLT5KGUUqrANHkopZQqME0e\nSimlCkyTh1JKqQLT5KGUUqrAyjg7gKLi7e0tjRo1cnYYSilVqmzduvWMiFQv6HrXTPJo1KgRISEh\nzg5DKaVKFWPMkcKsp5etlFJKFZgmD6WUUgWmyUMppVSBafJQSilVYJo8lFJKFZjDkocxZo4x5i9j\nzB+51BtjzLvGmFBjzE5jTFCGulHGmIMp0yhHxaiUUqpwHHnmMQ/om0f934DGKdM44AMAY0xVYCrQ\nFmgDTDXGeDkwTqWUUgXksOc8RGStMaZRHovcCnwmdhzcjcaYKsaY2kA3YIWInAMwxqzAJqEFjopV\nXQPi4iA2NvPk6QkNGtj6lSvh8mWSLsdy8XQMF/+K5WTFm9ldswenTkGbVTNIdilDYhkPEtw8SXT1\nIKJ6c07WDsJIMvWPriexjEemKdajCvFlK4AIXhUSqFM1ltpesdSqEkuNSrFUvqEaxrsaREXBb79l\nj69zZ2jeHI4ehY8/zv6ehgwBf38IC4O5cwFITobLlyHqMhwI/j8OlWlCwq59NN62iIQynpniO3RD\nL6Iq1KJ81Gm8zx0goYwHia4eJLrZ5aLK1yTZ1Q1EKO+eSLXysVQtF4uXRwxenrF4NPOhSnU3vC6E\n4x6+P3v8o0eDuzusWAG//po9/ueeAzc35IelJPy60a4WB3GxEBPnwuZ+04iMhJrrv6X6sW2ZVk0s\n48HaTv8AwHf3QmpE7M5UH+temfXtHwcgYMdnVDsfmqn+crnqbGrzEACtfv+YyhePZqq/UKkeW4PG\nAdBmyywqXD6dqf6c141sD7AXPdpvfAvP2POZ6iO8m7HLdzgAnX97FbeE6Ez1p2oGsKfZHQB0Wzsd\nl+TETPXH6rThwM0DMMlJdF/7QrZf3ZH6nTh0Y2/KJMTQ5bcZ2eoP+/QkvGFXPGIj6bDxnxwc/hxj\nJ7hlW86hRMRhE9AI+COXuv8BnTKUVwHBwBPAsxnmPwc8kcs2xgEhQEiDBg1EXYXERJFLl0QiIkSO\nHhU5eFBk1y47X0Rk3z6RJUtEFi0S+ewzkdmzRd59VyQpydavXSvy8cciX30lsnKlSEiIxPwRKmGH\nk2X9epFvvhF5/32R558XGTtWZOBAkdatRerXF3FzE/H0FKldW6R5c5EOHUT69RMZeVeCTJwo8o9/\niCy6f7ms+7/3ZfeQ5+Vo/3FyrssgOXf/ZDlzRiQhQSSpTj0RyDTtaDJU7rlHpFcvkQsuVbLVz2F0\nystkScJkq3+LR8QYkXJczlYnIC/yrBgjUpvjOdZPdnlD6tcXucNvf471h578UMLCRGLXb5VkY7JN\n/xn4Hxk1SmRy69WShMk29ed7AZEBLMlx+134RYwRGcn8HOuD2CrGiIzngxzrfTgkIDKFGTnWjxt8\nWh54QGRVh2dzjK/FjTFSvbrIOy5/z1YXh1vapv7NmGz1Z/ESY0SMEVnI0Gz14TRIq19K32z1u2iR\nVv8rHbPVr6ddWv02ArLVL6dXWn0oN2Sr/5rb0upPUz1b/TzuSauPxiNb/b94UIwRcSM+x9/dDJ4S\nY0S8OJdj/TO8JMaINOCIJGGkS5uYQv/rAyFSiOO7ses6RsqZx/9ExDeHuh+AGSKyLqW8CngS6AG4\ni8hLKfOfA6JF5M289hUcHCz6hHk+xMbCH3/A9u12evllqFwZXnkFnnkm+/IREeDtTeKUZyjz+ivZ\nqj9+N4YzUR50WTCBjrs+zFQXR1k8iAUMs5jIQL4nkipEuXkR71mFqMp1WdzzA2rWhHZb3qVO+AbK\nXTpFxcun8Io7xZEyN9K9QgiRkbAhqTWtCSEZw1/U4BS1WEM3HuVtAB7gAzyJIQZPYvEgFg+Ol2nE\nkTrtqV0bOrlvwat6GSrX9KBqHTtVa1SRmjdWoGZNcHdNzP7NumJFqFkTEhNhzZrMdTExEBgIrVsj\nl6OJe+1tLsZ7cCHOg/MxHpyN9mSPe0t2JTTlzLFYvMK3ceKcByfPexCTEl8kVYjFM9ePys0NatVK\nn2rXzvl1zZrg6ZaY+ewrJgbq1IFy5eDUKfuZZ31/t90G1arBpk0kLFtJjHgSnezB5SQPLiV4cLDJ\nAM7EVyIp/Cgux49yLsaDs5c9OBvlQUSUJ2GXa3DugivnzwkurgYvL6hShbSfGae85lWuDGXLXs0f\ntboaxpitIhJc4PWcmDw+AtaIyIKU8n7sJatuQDcRGZ/Tcrkprcnj7Fn46iuIj8/5H6xKFahQAYwp\nxMbPnbOXFcqXJ2H5z5hH/47rgb2YpCQAEjwrsuTxtRyqGEjF3RupcWBd2gEwMtaDczGeLEnqz+kL\nHnjHHqUGf6UdmFOnc1QFDDUrXKaJ91l8qkbSqHIkdctHUr18NOf73EWtWtBi0xy89/2KR2wkLhci\nITISypeHdetsrCNHwqZNmY+ON98MkyYhAtF/HOZCvCdnXaoTGVWGyJRNRGbYVNYDbZUqhfy9OVBi\nIvz1lz2ep04nT9qPKWtSqFq15MWfE5HSEafKWWlMHv2BSUA/7M3xd0WkTcoN861Aauur34FWknIP\nJDelKXmIwMaN8MEHsGiR/cIIcB//JpEymQ7OYfhwyLUJVSoLLcsfwL2yBx5VPChX1U4Vq5WlUmWD\nXIqi5h+r8D62nTp/bcPn/DZqxf/JBK8v+SxuGDdHb+NFnmM7gWyjJdsJJAwfJKXNRJky+fuWmNs3\nR8/cv0ArpUqwwiYPh90wN8YswJ5FeBtjjmFbULkBiMiHwFJs4ggFooF7U+rOGWNeBLakbGr6lRJH\naREVBV98YZPG0R1neaLse/yvWRQ1PnuDunWhaq0JmMTMN9Z+7ziJb7q9x6Uz8bzzUVPIfN+PmWWe\n4snEGTQyZwmTwSRjOOLehD1eHVhR/UGqNglkQiOoUqUl4VX+R6MqEJjDwb98ef32qJTKP4eeeRSn\nknzm8ccfNmHMnw8VLx3nVe83uevSbNziLtsWNYsW2SP3qVPZr0tXrw433mivd3z1Vfb6du1I6NSd\nMq6C2bwJ/PxsJlBKqXwocWce17u4OPj6a5s01q2z17TfCv6C8ZvuxZxPxgwfDlOmgG+GK3q1auW+\nwTJlYPjwHKtsAz0D7doV5VtQSqlcafIoYmFh8NFHMGeObag0sN425j5WhgFP++F9qQPMvB8mTwYf\nH2eHqpRShabJowgkJcGPP9qzjB9/BIPwVMdfebThDLxDlkH47eD9NXj7wPvvOztcpZS6apo8rtLZ\ns9CpE+zbZ5tZzh2xgrv2T8N93Xp7v+KVV2DCBGeHqZRSRUp71b1Kjz0GYQcTWfB5EkeOwKgmG3GP\nOA7/+hccOQJPP22bNCml1DVEk8dVWLYMvv/sHH96BXCX29e4uQGPPw4HD8KDD+rDD0qpa5Zetiqk\nS5dg/Hj4tNJDVI88YLuyANsdhFJKXeM0eRTSM89A8J/fMJD/wAsvwN/+5uyQlFKq2GjyKIT16+HL\n9yI45PkANAuy9zWUUuo6ovc8CiguDu6/H4ZW+5kKRMGnn9ruT5VS6jqiyaOAXn4Z9u6FgZ8Pw4SH\nZ35CXCmlrhOaPApg506Y88opZtyyir59gRo1nB2SUko5hSaPfEpMhPvGCB+7PsCUdQPsoAxKKXWd\n0hvm+fTOO9B06+f8jf/CG2/oWYdS6rqmySMfDh2CD549zvYyDyFtO2IeecTZISmllFPpZasrEIFx\nY4X3E8dSrkw8Zu5ccHV1dlhKKeVUDk0expi+xpj9xphQY8xTOdQ3NMasMsbsNMasMcbUy1CXZIzZ\nnjItcWSceZkzB35eDZ53DsTlnbehcWNnhaKUUiWGw0YSNMa4AgeAXsAx7LCyw0VkT4ZlvsKOcf6p\nMaYHcK+I3J1SFyUiFfK7P0eMJHjiBDRvDoGB8PPP4KLnaUqpa0xhRxJ05OGwDRAqIodFJB74Erg1\nyzLNgVUpr1fnUO80IvDgRGHOpaEsuHWBJg6llMrAkYfEusDRDOVjKfMy2gHckfL6NqCiMaZaStnD\nGBNijNlojBnswDhz9PXXUPO/H3F78mJql7tY3LtXSqkSzZGtrUwO87JeI3sC+JcxZjSwFjgOJKbU\nNRCRE8aYG4CfjTG7RORQph0YMw4YB9CgQYMiC/zcOXj9gcOscXmC5B69cBk3rsi2rZRS1wJHnnkc\nA+pnKNcDTmRcQEROiMjtItISeCZl3oXUupSfh4E1QMusOxCR2SISLCLB1atXL7LAn3gsmZlnx+Be\nzhWXOZ+AySkPKqXU9cuRyWML0NgY42OMKQvcBWRqNWWM8TbGpMbwNDAnZb6XMcY9dRmgI7CHYvDT\nT3Ds05V05Rdc33kL6te/8kpKKXWdcdhlKxFJNMZMApYDrsAcEdltjJkOhIjIEqAbMMMYI9jLVg+m\nrN4M+MgYk4xNcK9mbKXlKFFRdoAn9ya9iXt/He7dOzh6l0opVSo5rKlucSuKprqPPpzE0vdC+eTX\nJnTqVESBKaVUCVYSm+qWKhs3Qpn33mK3ix+dqhbLFTKllCq1tG8r7ABPL4/cy1c8i/QfAM2aOTsk\npZQq0fTMA3jt5USeOzQKl0oVcPv4A21dpZRSV3Ddn3kcPAjxL8+kDVvg34ugZk1nh6SUUiXedZ88\nbrwRbusfT2zZEXgMHerscJRSqlS47pOHiwu0WjLVdmallFIqX/SeRyq9z6GUUvmmyUMppVSBafJQ\nSilVYJo8lFJKFZgmD6WUUgWmyUMppVSBafJQSilVYJo8lFJKFZgmD6WUUgWmyUMppVSBOTR5GGP6\nGmP2G2NCjTFP5VDf0Bizyhiz0xizxhhTL0PdKGPMwZRplCPjVEopVTAOSx7GGFdgFvA3oDkw3BjT\nPMtibwCfiYg/MB2YkbJuVWAq0BZoA0w1xng5KlallFIF48gzjzZAqIgcFpF44Evg1izLNAdWpbxe\nnaG+D7BCRM6JyHlgBdDXgbEqpZQqAEcmj7rA0QzlYynzMtoB3JHy+jagojGmWj7XVUpdjeTk9Ncj\nRsCTT6aXd+2Cs2eLP6b8OHkSVq5ML0+YAIGB6e8nIaH4YgkLg/Xr08vbttkpVULCNdtjtyOTR07d\n1Gb9LT4BdDXGbAO6AseBxHyuizFmnDEmxBgTEhERcbXxKlU84uLSX+/YAbNnw/nzxRvDjBnQrl36\nga1KFahUKb3+b3+Dxx5LLz/1FCxdml7OmHgcbfNmeOIJSEqy5ffes/HFxtpynz7Qu7cdXwFgyBAY\nMCB9/dTlisK2bTB3bnr50Udh9Oj08tNP22SW6m9/g+7d08sTJsDkyenl99+HL79ML3/2Gfz4Y3p5\n7lxYvjy9/MknmRPn7NmwenWh385VERGHTEB7YHmG8tPA03ksXwE4lvJ6OPBRhrqPgOF57a9Vq1ai\nlNOdOiXy008icXG2vGKFyPDhIrGxtvzqqyKQXj9tmkjVqiIJCbb8v/+JLF4skpxctHFFRYnMmZMe\nxxdfiIwfL3L5cs7LL10qsmmTfR0fL1Kzpo01tVyxosjbb9tyYqLIBx+I7Nljy9HRIj/+KHL8uC3H\nxYns2ydy8aItJyXZODK+x0uX0n8nq1aJtGolcuyYLc+ZI+LhIRIebsuhoSIbN6b/zrL6179E3nkn\nvdy0qciDD2beV3799pvIww/bmEVEnnxSxN09PdZt20R27EhfftcukS1b0svz5onMnZtenjBBZPLk\n9HJAgMiQIenlJk1Ehg1LL/v4iNx9d3q5bl2R++5LL3t7i0ycmP/3kwMgRApzjC/MSvnasB1o6jDg\nA5TFXqJqkWUZb8Al5fXLwPSU11WBMMArZQoDqua1P00eqlgkJ4v8+adITIwt//CDyA03iBw+bMtz\n5th/q7AwW/7sM5Ebb7RJRURk3TqRl15KP2hHRIicPJm+/d69RYKD08vr1tllriZeEZvQQOTbbwu/\nrcRE+/PCBZHHH7cHeRGRQ4fstj/+OHP5009tee9eW16wwJZ37LDlxYtt+fffRYwRWb3aljdtErnl\nFrueiE00uSWKK0lKEnn55fR9xcSIeHqKvP66LScni5w/n778+vUi/fqJnD5ty//+t0ilSiJHj9ry\nqVMiZ88WLpbcZHxvkZGZk1tkpE38qc6fz17O7QtAPpW45GFjoh9wADgEPJMybzowKOX1EOBgyjL/\nBtwzrDsGCE2Z7r3SvjR5XMe+/VZk+fL0cnR00X1zj4y039JTv/WmHoRTD3Rbt4rceafIkSO2fOKE\nyNq1NobCSEiwySn1tbe3PXNJlZqEriQmRqRbN3umI2IPouvXF/0ZjYg9Ezl5Mv2gFxNj95V6AI6M\nFPnPf9J/hydPirzyij0bEbHJZvp0e0bhaJGRdl/r19vyvn3281y3zpbXrRNp3lxk+3ZbjotLP+u4\nRhU2eRi7bukXHBwsISEhzg5DFYf4eNi7FwICbLlTJ3u9PvWafFAQNGwI335ry48/Do0bwwMP2PIv\nv0CdOnZeKhE7muT58/b6/h132Ovohw/bge4/+gjGjYNz52DBArj1VqhXD4cSsdfY3dzAz4+E439R\npkFtQu55jxWNJxJ2WAgPh/AjhgoVoFP9I3Qov4MbHx1EmzbYa/GdO8N99zk2ztLs+HF7H6FXL2jf\n3tnROIUxZquIBBd0vet+DHNVSiQnp98QffhhewD/6y9wd4dFizIvO348eGV4LOj33zPXDx0Kt98O\nH35oy02awD33wDPPQIUKsGQJtGxp63x8YOdOaNrUlqtWhQcfLPK3J2LfTni4fZutWwPGcOsLQXTr\nBo/6QUxCGV5KnsGSeT3YD/Tz2sgn0aN4r8sidpcJpPvPz9H98vfM8TtFmzbuHHtpHl27wnu1oV8/\nm/f27rVvpVq1In8LpVPduvD8886OIk3qn/nly/bPrnFj8PaG0FD7Zx4ZmXk6f97+3LsXyhTz0VyT\nhyr5li+HMWNg40aoX9+2WBk0CFxdbX2dOpmXHz8+czlra5QffkhvWSRiv3XedJMtu7nBiRPpY9ob\nA35+aavGxOT8zxsdbRv1xMZCxYrpIbz5pv35+OP253332QSRumxsrN3mqVP2J9hwfvrJvnbJ0B6y\nUqOq9Fz2JPc2sCdW5XYamH4TM//TCKoAYS8QE/8S4+q4A/YErXVrqFEj/dcwZIh9Xb26TSLNmmVP\nJn/7m60PDbWtUG+/3ebUnTth+/YcPyGqVLF5tlEj+/6vVfHx6Z95xikqyn6WAQH2RDg6GqZOhYED\noUsXOHrUNl7L+LmnfvaxsXb9yEj79/Lgg3DoEHToAIsX25Pgw4ftdxsPD/u9qEoVO1WvbhNMfHzx\nJw+9bKVKnnPn4I037KWhtm3tUWzKFHj55bQzgKgo29z/1Ck7xcdn3sQdd9h/tB07YPdu+xgDwIYN\n9h8xIxH7TS/1QBAbC2+9ZeueecZeOUq9Ita1K6xdm3f4N90EBw/a17fdZvPPN9/Y8pAhNl5PTxtf\n6lSjhj3w+vjAzTfbqaidPQubNtlvqfv22Wnv3uyPc2zYYFvxzpljk92RI9Cggf31P/vslfcTFmbf\ny5Il9nf16qv2wBYRAeXKQfnyRf/echIXZ7/Je3ra8sGD6Uk+9aCdOqUevH187N8O2M+ud2/7XeXC\nBahdOz3B5+bxx+2f7qVLUKsWvPYaTJpkfycDB2b+zDNO5cvbpDBokE0aUVHw22/2BLhGDfu4SFKS\nXbaoFfaylSYPVTIcOkTi+Uv8VSeQ04cvE9i/Dmb6dDa2/TtffAGvvGK/0b72Grz4oj3Y5+X0aftP\n9+yz9uCVmGjnjxmTuZl+Vm5u9lv48eP2W/8779gDbOoVrsWL4cyZ9G9+qVOFCpkPBsX9LfBqnDkD\nFy+ml+vUse/h4kVb16CBfT/nz+f8OIqITUDh4fYg+dhj9vc4fbr9vZ04YZcbMcJebaxePT1RNmpk\np/Ll7UG8bNn0xybmzLEH0YcftuUnn7TfI7Ie+DNOAQGwbJld3tfXftdYvNiWq1Wz30vyMnhw+q2y\n3r3t5b5HHrEH7qeeyvytP+vn7+mZ/rM00eShyaNkOXnSfl1LvVewbJk98gwfzqVLcPbxVzgTdokP\nGsxgx3Zh0Y6bOZzUkF7YB6DWLb9Mx97l+eILeOgh2LrVHmyWLYMVK+y3utSpZs3s/7A+PvaAd+aM\n3W3qvfHTpzMfKFOVL28PAp6e6Ves1NVLbYcA9nMLCUlPMuHh9qwm41ljnTo2cYM9kJ87l36md9tt\n9nJOTt/cU8/kbrop/Rm8uXPtwX7wYFv+7jv7M3Udd/f01xUq2GXLl7/+Pn9NHpo8rs727XZK/dq3\ncqW9x5B6nWLBAvtf/MEHtvz227BmTfp/5COP2IvqO3bY8l13we+/E739AGvWQLd/DaHckb0snbmb\n/v3hAx7Ai/NM8l5Iy5bQq9xvuPo0oHzT+tSqZa8bV6uW+eCjrj3JyfZ7RmxsehKoWtXZUV1ftLWV\nKpjdu23XBjNn2msF330HL7xgWx25uMDPP9uLt6nJIzQ088V+kUxdVCQHBXM+tjwrF9oc1DvwEbqP\njuT8eejfH2a/OpuxE8oQcMledqrf8kMCA+GvOqnJoWOOYWriuLa5uNgGT6r00TOP64WIvfbTqJFt\n+/fDD3DnnfDrr/a5iHPn7F2+Bg3sETspyf5nG5N2g9HLy27mww/Tb1SfPGmnvXvT70O4udkbhzNm\n2OV/+81ei76WW+EoVVrpZStNHjlLSLBH84MHbROet9+Gv/+d5LgEzp6M59Sl8mlJIOPP2rXhn/+0\nm2jSxHZaunChLVetalumVK+eft+haVPbMiQwEJo3tyczSqmSTy9bKcB++z91Cs6dSab1Ix0hOJhP\ng9/j6NHGPLtoEfTsSZ8+sGqVG0lJbtnWL1/eJo527dLnPfts5ucA9u2zCaQ0tShSShUt/fcv5fbs\ngX/8w/7scvQLbo7dwRRex9PThcuP3YK56UZ++cUOz/DslqGAfQgsONieMdSubVsr1a5tyxUqZN/H\n3XdnLqc+dKaUun5p8iilYmPtQ1tvvpqAZyU3evWCuyr/QYuTP1PrhQRq1nNDer2IcYFPRmW+8fzI\nI86LWyl1bdDkUQodOWK7sIg9+CehlXpT/q3XqHzPrZAwHdxmcE+W5bXFklKqqDlyJEFVxFLbNtSt\na7tbmvvfatTpeAOVb6puK9yy38NQSilH0ORRSixbZpu7njsHZTau4+sF8fQcVN52utShg7PDU0pd\nZzR5lBK1atmOYKO2h0K3bvYhCqWUchKHJg9jTF9jzH5jTKgx5qkc6hsYY1YbY7YZY3YaY/qlzG9k\njIkxxmxPmT50ZJwlUUKCffh74kRbDgy0z/M16HET/Oc/6X18K6WUEzgseRhjXIFZwN+A5sBwY0zz\nLIs9CywSkZbAXcD7GeoOiUhgyvSAo+IsibZsseMwPPmk7ZE04XI8TJqE2bPbLnDnnTm3qVVKqWLi\nyDOPNkCoiBwWkXjgS+DWLMsIkDIqD5WBEw6Mp8S7dAn+/nc7hEVEhB0D4rvvwC0ywvYTvWqVs0NU\nSinAsU116wJHM5SPAW2zLDMN+MkY8xBQHrglQ52PMWYbcBF4VkR+dWCsTrdkiR1B7Phxe6nq5Zeh\nMhdAKtnmVbt32z7DlVKqBHDkmUdOTxdk7UhrODBPROoB/YD5xhgX4CTQIOVy1mPAf4wxlbKsizFm\nnDEmxBgTEhERUcThF4+YGBg2zA6aV6WKHfbzX/+CylHHbXvcd96xC2riUEqVII5MHseA+hnK9ch+\nWeo+YBGAiGwAPABvEYkTkbMp87cCh4BsA3OKyGwRCRaR4OrVqzvgLTjeG2/AV1/BSy/B779n6FOq\ndu30AZCVUqqEcWTy2AI0Nsb4GGPKYm+IL8myzJ9ATwBjTDNs8ogwxlRPueGOMeYGoDGQZeTpa8OU\nKfYZjmeeSXnG7/vv7cMcLi4wa5btLl0ppUoYhyUPEUkEJgHLgb3YVlW7jTHTjTGDUhZ7HBhrjNkB\nLABGi+0jvguwM2X+YuABEbnC6MOlR1ISTJtmh0gtW9aOlQzY/tDvvNMO/qyUUiWYjufhBDt22MtT\n77wD48ZlqVy3znZ56+HhlNiUUteXwo7noU+YO0FAgG08NW4cdijXkSNh+XJb2amTJg6lVImnyaOY\niNhxN+bMseUbbkipOHsWDhyw2UQppUoJTR7F5MUXbXdUW7dmqaheHTZsgEcfdUpcSilVGJo8isHM\nmTB1KowaBe+9lzJTBGbPtuPGurrqoBtKqVJFk4eDzZpl+6gaNgw++cS2wAVg82YYP952cqiUUqWM\njiToQHPmwKRJ9unx+fPtCUaatm1h0yZo1cpp8SmlVGHl+8zDGNPJGHNvyuvqxhgfx4VV+i1YAPff\nb5/hWLgwyyB/J0/an23aZMkoSilVOuQreRhjpgJTgKdTZrkBnzsqqNLuu+/g7rttzyLffgvu7hkq\nv//eNrVav95p8Sml1NXK75nHbcAg4DKAiJwAKjoqqNKuShXo0cPmiXLlslQGB9tuc4ML/EyOUkqV\nGPm95xEvImKMEQBjTHkHxlRqnT4NNWvaUWK7ds2lAVXt2vDmm8UdmlJKFan8nnksMsZ8BFQxxowF\nVgIfOy6s0mfnTrjpJvg85WJetsTxyy8wdKh9KFAppUq5fJ15iMgbxphe2IGZmgDPi8gKh0ZWyjRp\nAmPGQM+euSwQGgp79oCnZ7HGpZRSjnDFjhFTukZfLiK35Lmgk5WKjhETErI0u1JKKedyWMeIIpIE\nRBtjKhcqsmvcjh1Qv769KpXrAr+mjKCriUMpdY3I7w3zWGCXMWYFKS2uAETkYYdEVYrs2wfHjoGX\nVy4LTJu0MWz0AAAgAElEQVQGW7bAoUNZ2uwqpVTpld/k8UPKpLLYv9/eHG/cOJcF5s+39zs0cSil\nriH5vWH+acpQsqnjiO8XkQTHhVV67NsHDRvmcB/8zBl7OlKhAgQGOiU2pZRylPw+Yd4NOAjMAt4H\nDhhjuuRjvb7GmP3GmFBjzFM51Dcwxqw2xmwzxuw0xvTLUPd0ynr7jTF98v2Oitn+/balVSbJyXDb\nbTBwoFNiUkopR8vvZas3gd4ish/AGHMzdszxXHv1S2mlNQvoBRwDthhjlojIngyLPYsd2/wDY0xz\nYCnQKOX1XUALoA6w0hhzc8rN+xJDxCaPzp2zVBgDEyZk6EJXKaWuLflNHm6piQNARA4YY67UdKgN\nECoihwGMMV8CtwIZk4cAlVJeVwZOpLy+FfhSROKAMGNMaMr2NuQz3mJx/LgdjiPbmYcxMGKEU2JS\nSqnikN+vxiHGmE+MMd1Spo+BrGPiZVUXOJqhfCxlXkbTgJHGmGPYs46HCrAuxphxxpgQY0xIRERE\nPt9K0dmfkk7TkocIjB4NX31V7LEopVRxym/ymADsBh4G/o49e3jgCuvk1LNT1icShwPzRKQe0A+Y\nb4xxyee6iMhsEQkWkeDq1atfIZyit2+f/ZmWPC5cgL174cSJXNdRSqlrQX4vW5UB3hGRf0La/Ywr\ntT09BtTPUK5H+mWpVPcBfQFEZIMxxgPwzue6TtewIdx1F9SpkzKjShXtal0pdV3I75nHKiBjY1RP\nbOeIedkCNDbG+KQ0870LWJJlmT+BngDGmGaABxCRstxdxhj3lEGnGgOb8xlrsRkwwA76ZAy2R8TU\n8ch1gCel1DUuv8nDQ0SiUgspr7OOVJGJiCQCk4DlwF5sq6rdxpjpxphBKYs9Dow1xuzAtt4aLdZu\nYBH28tgy4MGS1tIKICr1N/LHH3b0pw8/dGo8SilVXPJ72eqyMSZIRH4HMMYEAzFXWklElmJvhGec\n93yG13uAjrms+zLwcj7jK3bR0VCpEsycCY8/7msvV+l45Eqp60R+k8cjwFfGmBPYG9d1gGEOi6oU\nSEyEV17J8IxH+/ZOjUcppYpTnpetjDGtjTG1RGQL0BRYCCRiLyWFFUN8JValSvDUU9Dm2Df2ktWl\nS84OSSmlis2V7nl8BMSnvG4P/AP71Ph5YLYD4yrxwsPtQ4KcPAlbt9o+rJRS6jpxpeThKiLnUl4P\nA2aLyNci8hxwk2NDK9n+8Q/o2BF48EE7QmCOA5YrpdS16YrJwxiTel+kJ/Bzhrr83i+5Ju3fD02b\nOjsKpZRyjisljwXAL8aY/2JbV/0KYIy5Cbjg4NhKrNQOEbtV2wXt2kFJH/5WKaWKWJ5nDyLysjFm\nFVAb+EnSBzx3Ib0fqutOaoeIjevFwHEP+2S5UkpdR6546UlENuYw74BjwikdUjtE9OrTBl5b49RY\nlFLKGXTAiULI1iGiUkpdZzR5FML+/VC5fCJ12tXXLkmUUtel67rFVGHt2weBN0VhWveFRo2cHY5S\nShU7TR6FYIeerQIff+zsUJRSyik0eRTCrFngXTGOKw9popRS1yZNHoUwYADQsYcdBUqHnFVKXYc0\neRTQwYMQFgY9htxFmWqVnR2OUko5hSaPAlqwAKZNg8uXH6KM5xUXV0qpa5JDm+oaY/oaY/YbY0KN\nMU/lUP+WMWZ7ynTAGBOZoS4pQ13W4Wud5sEHYdPSs3heeSwspZS6ZjnszMMY44rtvr0XcAzYYoxZ\nkjJ6IAAi8miG5R8CWmbYRIyIBDoqvsKqVg2qLXsRhs2F8+fBRR+VUUpdfxx55GsDhIrIYRGJB74E\nbs1j+eHYjhhLLBGYMQMO+N0Br7+uiUMpdd1y5NGvLnA0Q/lYyrxsjDENAR8yd/nuYYwJMcZsNMYM\ndlyY+Xf8uB3HY1V8Zxg/3tnhKKWU0zjyhnlOoyNJDvMA7gIWi0hShnkNROSEMeYG4GdjzC4ROZRp\nB8aMA8YBNGjQoChiztP+/VCF8wS5n4CkpuDq6vB9KqVUSeTIM49jQP0M5XrAiVyWvYssl6xE5ETK\nz8PAGjLfD0ldZraIBItIcPXq1Ysi5jzt2wf9WErb+3zt6IFKKXWdcmTy2AI0Nsb4GGPKYhNEtlZT\nxpgmgBewIcM8L2OMe8prb6Aj4PSj9f79sKVcN2Tep9CsmbPDUUopp3HYZSsRSTTGTAKWA67AHBHZ\nbYyZDoSISGoiGQ58mWGgKYBmwEfGmGRsgns1YystZ9m/Hyo1q4sZdY+zQ1FKKady6EOCIrIUWJpl\n3vNZytNyWG894OfI2Apj/95kHvb5HiI6QDFcJlNKqZJK25rmU3Q0lD0aymNrB8OSEvPMolJKOYUm\nj3w6eBCO0JCVL26AgQOdHY5SSjmV9m2VT6GhEI87NQa1gxrOjkYppZxLzzzy6Y474MJHC2h+eYuz\nQ1FKKafTM4/8EqHSlIkwbBi0b+3saFQJlJCQwLFjx4iNjXV2KEpl4+HhQb169XBzcyuS7WnyyKd/\nPGNo9epB7hgQ5+xQVAl17NgxKlasSKNGjTAmpw4WlHIOEeHs2bMcO3YMHx+fItmmXrbKp++/h9/2\ne0PdHLvnUorY2FiqVaumiUOVOMYYqlWrVqRnxXrmkU+7XvgGOR8JjHF2KKoE08ShSqqi/tvUM4/8\nmj8f8/4sZ0ehlFIlgiaPfFiwAAbEf0PUdyudHYpSeQoPD8fX19eh++jXrx+RkZFXXjCftm/fztKl\nS6+8YBYnTpxgyJAhRRaHKhhNHvmwYQP8stZQvp6Xs0NRyumWLl1KlSpVimx7eSWPxMTEXNerU6cO\nixcvLrI4CispKenKC12DNHnkg9uGtXxQ7nFM5Hlnh6LUFSUmJjJq1Cj8/f0ZMmQI0dHRTJ8+ndat\nW+Pr68u4ceNI7Yd0y5Yt+Pv70759eyZPnpx21hIdHc2dd96Jv78/w4YNo23btoSEhADQqFEjzpw5\nQ3h4OM2aNWPs2LG0aNGC3r17ExMTk+d2s4qPj+f5559n4cKFBAYGsnDhQqZNm8a4cePo3bs399xz\nD+Hh4XTu3JmgoCCCgoJYv349kPksa968edx+++307duXxo0b8+STT+b6+0lKSmL06NH4+vri5+fH\nW2+9BUBoaCi33HILAQEBBAUFcejQIUQkLX4/Pz8WLlwIwJo1a+jevTsjRozAz892w/f555/Tpk0b\nAgMDGT9+/LWfVETkmphatWoljvKM1yyJLlNRJCbGYftQpd+ePXvSXv/97yJduxbt9Pe/XzmGsLAw\nAWTdunUiInLvvffKzJkz5ezZs2nLjBw5UpYsWSIiIi1atJDffvtNRESmTJkiLVq0EBGRmTNnyrhx\n40REZNeuXeLq6ipbtmwREZGGDRtKRESEhIWFiaurq2zbtk1ERIYOHSrz58/Pc7s5mTt3rjz44INp\n5alTp0pQUJBER0eLiMjly5clJuV/78CBA5L6vx4WFpa23blz54qPj49ERkZKTEyMNGjQQP78888c\n9xcSEiK33HJLWvn8+fMiItKmTRv55ptvREQkJiZGLl++LIsXL5ZbbrlFEhMT5dSpU1K/fn05ceKE\nrF69WsqVKyeHDx8WEfvZDxgwQOLj40VEZMKECfLpp5/m+p6dJePfaCpsL+cFPubqmccVREfDy+cn\n8uaz58HDw9nhKHVF9evXp2PHjgCMHDmSdevWsXr1atq2bYufnx8///wzu3fvJjIykkuXLtGhQwcA\nRowYkbaNdevWcddddwHg6+uLv79/jvvy8fEhMDAQgFatWhEeHp7ndvNr0KBBeHp6Avbhy7Fjx+Ln\n58fQoUPZk8tAbD179qRy5cp4eHjQvHlzjhw5kuNyN9xwA4cPH+ahhx5i2bJlVKpUiUuXLnH8+HFu\nu+02wD5QV65cOdatW8fw4cNxdXWlZs2adO3alS1bbC8Tbdq0SXtmYtWqVWzdupXWrVsTGBjIqlWr\nOHz4cIHfd2miTXWv4OBB+/PmZjrkrMq/t9923r6zNsk0xjBx4kRCQkKoX78+06ZNIzY2Nu3SVU7y\nqsvI3d097bWrqysxMTH5Xjcv5cuXT3v91ltvUbNmTXbs2EFycjIeuXyJyxpLbvdLvLy82LFjB8uX\nL2fWrFksWrSIt3P5wPJ6LxljFBFGjRrFjBkz8nxf1xI987iCI78dYwkDaZmofVqp0uHPP/9kwwY7\nMOeCBQvo1KkTAN7e3kRFRaXdZPby8qJixYps3LgRgC+//DJtG506dWLRokUA7Nmzh127duV7/3lt\nNycVK1bk0qVLudZfuHCB2rVr4+Liwvz586/6XsKZM2dITk7mjjvu4MUXX+T333+nUqVK1KtXj+++\n+w6AuLg4oqOj6dKlCwsXLiQpKYmIiAjWrl1LmzZtsm2zZ8+eLF68mL/++guAc+fO5Xrmc61waPIw\nxvQ1xuw3xoQaY57Kof4tY8z2lOmAMSYyQ90oY8zBlGmUI+PMy+mdp7mBw9Svd/XfppQqDs2aNePT\nTz/F39+fc+fOMWHChLTLPoMHD6Z16/S+2T755BPGjRtH+/btEREqV64MwMSJE4mIiMDf35/XXnsN\nf3//tLr8yG27OenevTt79uxJu2Ge1cSJE/n0009p164dBw4cyPSNvzCOHz9Ot27dCAwMZPTo0Wln\nC/Pnz+fdd9/F39+fDh06cOrUKW677Tb8/f0JCAigR48evP7669SqVSvbNps3b85LL71E79698ff3\np1evXpw8efKq4izpTFGcYua4YWNcgQNAL+AYdkzz4ZLLcLLGmIeAliIyxhhTFQgBggEBtgKtRCTX\n5k7BwcGS2hqkKP3f/8H69RAWVuSbVteYvXv30qyUjW0fFRVFhQoVAHj11Vc5efIk77zzDklJSSQk\nJODh4cGhQ4fo2bMnBw4coGzZsle1XeVcOf2NGmO2ikhwQbflyHsebYBQETkMYIz5ErgVyG0s8uHA\n1JTXfYAVInIuZd0VQF9ggQPjzVGNGtCjR3HvVani8cMPPzBjxgwSExNp2LAh8+bNA2xT3e7du5OQ\nkICI8MEHH+Q7ceS1XXXtcGTyqAsczVA+BrTNaUFjTEPAB/g5j3WLv0dCEd7a0hnuvhsYX+y7V8rR\nhg0bxrBhw7LNr1ixIldzJp/TdpcvX86UKVMyzfPx8eHbb78t9H6upG3btsTFZe4Je/78+WnPZqjC\nc2TyyKkXrtyukd0FLBaR1Dth+VrXGDMOGAfQoEGDwsSYt5gY8PaGlCaDSqnC69OnD3369CnWfW7a\ntKlY93c9ceQN82NA/QzlesCJXJa9i8yXpPK1rojMFpFgEQmuXr36VYab3epN5bhh53ds97+nyLet\nlFKlmSOTxxagsTHGxxhTFpsglmRdyBjTBPACNmSYvRzobYzxMsZ4Ab1T5hWrCuWFtm0hh8YVSil1\nXXPYZSsRSTTGTMIe9F2BOSKy2xgzHfs4fGoiGQ58KRmafYnIOWPMi9gEBDA99eZ5cWr9+lAWuLlB\nrWK/T6+UUiWaQ58wF5GlwNIs857PUp6Wy7pzgDkOCy4f4nxb4e6pz1EqpVRWemTMQ5N5TzPu0JQr\nL6hUCVEax/MoqHnz5jFp0iSn7V9Z2rdVLqLPx3HkSFnq19dhRZXKqDADN10rEhMTKVNGD5ugZx65\nuvjMa5zBm2Y3xF15YaVKkNI0ngdAbGws9957L35+frRs2ZLVq1fnOR/g6NGj9O3blyZNmvDCCy/k\nuu3Lly/Tv39/AgIC8PX1Tev+ZMuWLXTo0IGAgADatGnDpUuXct3fvHnzGDp0KAMHDqR3794AzJw5\nk9atW+Pv78/UqVNz3f+1TFNoLvZ7teMXEhns537lhZXKQbduV15mwAB44on05UePttOZM5B1hNU1\na/K33/379/PJJ5/QsWNHxowZw/vvv8+kSZN4/nl7u/Huu+/mf//7HwMHDuTee+9l9uzZdOjQgaee\nSu9+7v3338fLy4udO3fyxx9/pHW7ntXBgwdZsGABH3/8MXfeeSdff/01I0eOzHW7OZk1axYAu3bt\nYt++ffTu3ZsDBw7kOh9g8+bN/PHHH5QrV47WrVvTv39/goOz97CxbNky6tSpww8//ADYThbj4+MZ\nNmwYCxcupHXr1ly8eBFPT8+07lNy2t+GDRvYuXMnVatW5aeffuLgwYNs3rwZEWHQoEGsXbuWLl26\n5O8DukbomUcufnHvzTQzncaNnR2JUgVT2sbzWLduHXfffTcATZs2pWHDhhw4cCDX+QC9evWiWrVq\neHp6cvvtt7Nu3boct+3n58fKlSuZMmUKv/76K5UrV2b//v3Url07rYPISpUqUaZMmSvur2rVqgD8\n9NNP/PTTT7Rs2ZKgoCD27dvHwdSxG64jeuaRk5gYTuyIomHD6vpwuSq0/J4p5LS8t3fB109V2sbz\nyG35vLaT03vMyc0338zWrVtZunQpTz/9NL1792bw4ME5Ll+QsTuefvppxo+/vrss0jOPnKxZw4ff\n1OC2Gr85OxKlCqy0jefRpUsXvvjiCwAOHDjAn3/+SZMmTXKdD7BixQrOnTtHTEwM3333XdqZVlYn\nTpygXLlyjBw5kieeeILff/+dpk2bcuLEibQRAS9dukRiYmKe+8uoT58+zJkzh6ioKMB28Z46jsf1\nRM88ciBNmvJM2TcoE5TzqbpSJVnqeB7jx4+ncePGTJgwgfPnz+Pn50ejRo2yjecxduxYypcvT7du\n3TKN55F6071ly5aFGs8jp+3mZOLEiTzwwAP4+flRpkwZ5s2bh7u7e67zwSa3u+++m9DQUEaMGJHj\n/Q6w9y8mT56Mi4sLbm5uab0DL1y4kIceeoiYmBg8PT1ZuXJlnvvLqHfv3uzdu5f27dsDUKFCBT7/\n/HNq1KiR79/PtcBh43kUt6Icz+P4cahXD95/HyZMKJJNquuAjudx5e0q5yot43mUWp67NvPqcy3o\n3PnqRixTqqTT8TxUYemZR1ZnzkD16jBzZnobSqXyoTSeeRQnR47ncfbsWXr27Jlt/qpVq6hWrdpV\nb/9aoWcejlSuHCc/WoJbYAu8nR2LUtcQR47nUa1aNbZv3+6QbaucaWurrMqVY/TXA+k78QZnR6KU\nUiWWnnlktXo1L99ThzPVsjfRU0opZWnyyGrMGIJbt4aUNu5KKaWy0+SRxZn5P/LHzmSCLkKlSs6O\nRimlSia955HFzyea0v3B5oSHOzsSpZwvtQddR4iLi+OWW24hMDAwrbfborZ9+/YrdiG/Zs0a1q9f\nX+Bth4SE8PDDDxc2tFLPocnDGNPXGLPfGBNqjMmxa01jzJ3GmD3GmN3GmP9kmJ9kjNmeMmUb+9wh\n1q6Fb77BINohoir1RITk5GRnh5Grbdu2kZCQwPbt2xk2bFi+1klKSirQPq42eSQmJua6XnBwMO++\n+26B4nGEvGJ0KBFxyIQdt/wQcANQFtgBNM+yTGNgG+CVUq6RoS6qIPtr1aqVXLVhw+R0eR9p2PDq\nN6WuP3v27Mk8o2vX7NOsWbbu8uWc6+fOtfUREdnr8iEsLEyaNm0qEyZMkMDAQBk9erS0atVKmjdv\nLs8//3zacg0bNpTnn39eWrZsKb6+vrJ3714RETlz5oz06tVLAgMDZdy4cdKgQQOJiIgQEZE333xT\nWrRoIS1atJC33norbX9NmjSR++67T1q0aCEjRoyQFStWSIcOHeSmm26STZs25Rjn6dOn5cYbb5RK\nlSpJQECAhIaGysqVKyUwMFB8fX3l3nvvldjY2LRYX3jhBenYsaMsWLBAQkNDpU+fPhIUFCSdOnVK\ni33RokXSokUL8ff3l86dO0tcXJzUr19fvL29JSAgQL788sscf181a9aUOnXqSEBAgKxdu1ZGjRol\njz76qHTr1k0ee+wx2bRpk7Rv314CAwOlffv2sm/fPhERWb16tfTv319ERKZOnSr33nuvdO3aVXx8\nfOSdd97J9TOKioqSfv36ib+/v7Ro0SItrs2bN0v79u3F399fWrduLRcvXpSYmBgZPXq0+Pr6SmBg\noPz8888iIjJ37lwZMmSIDBgwQLp37y4iIq+//roEBweLn59fps86o2x/oyIChEhhjvGFWSlfG4b2\nwPIM5aeBp7Ms8zpwfy7rF3/yiImRQS1CpU+fq9+Uuv6UlORhjJENGzaIiMjZs2dFRCQxMVG6du0q\nO3bsEBF7QH733XdFRGTWrFly3333iYjIQw89JC+88IKIiPzvf/8TQCIiIiQkJER8fX0lKipKLl26\nJM2bN5fff/9dwsLCxNXVVXbu3ClJSUkSFBQk9957ryQnJ8t3330nt956a66xZjz4xsTESL169WT/\n/v0iInL33XenJaiGDRvKa6+9lrZejx495MCBAyIisnHjxrSDp6+vrxw7dkxERM6fPy8i9iD74IMP\n5vk7mzp1qsycOTOtPGrUKOnfv78kJiaKiMiFCxckISFBRERWrFght99+e7b4p06dKu3bt5fY2FiJ\niIiQqlWrSnx8fI77W7x4sdx///1p5cjISImLixMfHx/ZvHlzpn2+8cYbMnr0aBER2bt3r9SvX19i\nYmJk7ty5Urdu3bTPd/ny5TJ27FhJTk6WpKQk6d+/v/zyyy/Z9l2UycORN8zrAkczlI8BbbMsczOA\nMeY37JnKNBFZllLnYYwJARKBV0Xku6w7MMaMA8YBNGjQ4KoDFncPfj5yI2OyP6iqVMHl1ad6uXJ5\n119Fn+wNGzakXbt2ACxatIjZs2eTmJjIyZMn2bNnT9rYHLfffjtgx+H45ptvAFi7dm3a6/79++Pl\n5QXYMTduu+22tK7Jb7/9dn799VcGDRqEj48Pfn5+ALRo0YKePXtijMHPz4/wfN483L9/Pz4+Ptx8\n880AjBo1ilmzZvHII48ApF3WioqKYv369QwdOjRt3bg4O9pnx44dGT16NHfeeWfaeyusoUOH4urq\nCtgBpEaNGsXBgwcxxpCQkJDjOv3798fd3R13d3dq1KjB6dOnqVevXrbl/Pz8eOKJJ5gyZQoDBgyg\nc+fO7Nq1K9sYI2B/7w899BCQ/zFGUn9PBw8edOgAVY5MHjl1sJ+1L5Qy2EtX3YB6wK/GGF8RiQQa\niMgJY8wNwM/GmF0icijTxkRmA7PBdk9yVdHu2sXF//xA2aixNGmi3Rmo0iv1AB8WFsYbb7zBli1b\n8PLyYvTo0cTGxqYtl9pjrKura6br5gUd6yJjz7MuLi5pZRcXl3xfj89r+5D+npKTk6lSpUqOT5N/\n+OGHbNq0iR9++IHAwMCreuI84/gdzz33HN27d+fbb78lPDycbrkMEZl1bJPc3vu1MsaII2+YHwPq\nZyjXA07ksMx/RSRBRMKA/dhkgoicSPl5GFgDtHRgrPDbb1R+9WkMQg5d+CtV6ly8eJHy5ctTuXJl\nTp8+zY8//njFdTKOafHjjz9y/vz5tPnfffcd0dHRXL58mW+//ZbOnTsXWaxNmzYlPDyc0NBQAObP\nn0/Xrl2zLVepUiV8fHz46quvAHvQ3LFjBwCHDh2ibdu2TJ8+HW9vb44ePUrFihW5dOlSnvu+0jIX\nLlygbt26AEXSweO1MsaII5PHFqCxMcbHGFMWuAvI2mrqO6A7gDHGG3sZ67AxxssY455hfkdgjwNj\nhQce4N8zz3MWb5o2deielCoWAQEBtGzZkhYtWjBmzJhcB0zKaOrUqaxdu5agoCB++umntMvBQUFB\njB49mjZt2tC2bVvuv//+tEskRcHDw4O5c+cydOhQ/Pz8cHFx4YEHHshx2S+++IJPPvmEgIAAWrRo\nwX//+18AJk+ejJ+fH76+vnTp0oWAgAC6d+/Onj178mwOPHDgQL799lsCAwP59ddfs9U/+eSTPP30\n03Ts2LHArb1ysmvXLtq0aUNgYCAvv/wyzz77bKYxRgICAujVqxexsbFMnDiRpKQk/Pz8GDZsWJ5j\njIwYMYL27dvj5+fHkCFDrpg0r5ZDe9U1xvQD3sbez5gjIi8bY6Zjb9AsMfY87U2gL5AEvCwiXxpj\nOgAfAcnYBPe2iHyS176Kolfdhx+GuXPh4kXIZVRLpXKlveqqkq7U9KorIkuBpVnmPZ/htQCPpUwZ\nl1kP+DkytkwuXoRHH2XqnRMYMSJYE4dSSl2Bdk8CEB4OS5ZQbcgQqrVzdjBKXVvmzp2bbRTBjh07\nMmvWrGs6jmt9jBEdDCpFTLTw7jvCrbe56D0PVSh62UqVdEV52Ur7tkpx6LDhqX+4sHOnsyNRSqmS\nTy9bAYwciW+PHkRGjsHNzdnBKKVUyafJIzERjhyBs2epXNnZwSilVOmgl63KlIFff+WdspN56y1n\nB6OUUqWDJo8Un30Gy5c7OwqlSpbrYTyPgpo2bRpvvPFGkW6zNNLLVoAIHDgAnTo5OxKlik5q76cu\nLiXzO2LG8TzyKykpKa3DwvzYvn07ISEh9OvXrzAhljiJiYmUKVMyDtsl86+qmJ04AVFRaJ9Wqmh1\n6wapfSElJNjy55/bcnS0Lad+475wwZZTerTlzBlb/v57Wz51Kl+7DA8Pp1mzZkycOJGgoCDuu+8+\ngoODadGiBVOnTk1brlGjRkydOpWgoCD8/PzYt28fYJ9N6N27Ny1btmT8+PGZOub75z//ia+vL76+\nvrz99ttp+2vatCn3338/vr6+/N///R8rV66kY8eONG7cmM2bN+cY519//cXIkSPZvn07gYGBHDp0\niFWrVtGyZUv8/PwYM2ZMWm+5jRo1Yvr06XTq1ImvvvqKQ4cO0bdvX1q1akXnzp3TYv/qq6/w9fUl\nICCALl26EB8fz/PPP8/ChQvzPLs5d+4cgwcPxt/fn3bt2rEzpcllbvMBduzYQY8ePWjcuDEff/xx\nrp/HyZMn6dKlC4GBgfj6+qZ1f7Js2TKCgoIICAhIexYkt/1NmzaNcePG0bt3b+655x6SkpKYPHky\nrVu3xt/fn48++ijX/TtUYfpxL4nT1YznsXKlHdlk1apCb0KpnMfzSB2fIz7elufPt+XU8TxSByiK\njJcCSdIAAAwNSURBVLTlr7+25dTxPJYsseWTJ/MVg47nUfDxPCZNmiTTpk0TEZFVq1ZJQEBAnvOn\nTp0q/v7+Eh0dLREREVKvXj05fvx4jtt+44035KWXXkr7DC5evCh//fWX1KtXTw4fPiwi6Z9RXvsL\nCgqS6OhoERH56KOP5MUXXxQRkdjYWGnVqlXatq6ktIznUWrs329/6pmHKlIZx+Nwc8tczjqeR+XK\nmctZx/OoVSvfu9XxPAo2nse6dev4+uuvAejRowdnz57lwoULuc4HuPXWW/H09MTT05Pu3buzefNm\nBg8enG3brVu3ZsyYMSQkJDB48GACAwNZs2YNXbp0wcfHByBtTI689jdo0CA8PT0BO3bHzp07Wbx4\nMWB7/T148GDa9oqLJg9g3z6oUAHq1HF2JEpdPR3Po2DjeeS0b2NMrvMz/sw6P6suXbqwdu1afvjh\nB+6++24mT55MlSpV8v07Tl0u69gd7733Hn369MnjXTme3vPAnnk0aaI96apri47nkb/xPDK+5zVr\n1uDt7U2lSpVynQ/w3//+l9jYWM6ePcuaNWvSRgDM6siRI9SoUYOxY8dy33338fvvv9O+fXt++eUX\nwsLCAHuvI684surTpw//3979B1lV1nEcf3/ElRWp0FicZEVcwtRxWFxlR9NEwVDDSYfxB+tvFC1S\nUydppJFCowklSysjCX/AjFmkDmI1wY6BOU6uoKggBhgSbpHsrPiLTHfbb388z+rxsj/u3d3LLud8\nXzMMe557zrnPc597z/ec59z7fOfNm/dRRsONGzeyc+fODttYDH7lQbjy8G9aubRJ5vOoqKjIO59H\nTU0NVVVVjB07ts18HsBH+TzyHZbqTDKfR3NzM2PGjOkwn8e0adOYPXs2TU1NTJ48mcrKSqZPn86m\nTZswM8aPH09lZSXDhg1jzpw5jB49mhkzZnw0/JU0a9YspkyZwqhRoxgwYAALFy7ssBygurqaiRMn\nsnXrVmbOnMlB7QxbrFy5krlz51JSUsLAgQNZtGgRZWVlzJ8/n0mTJtHS0sKQIUOora3t8PmSpk6d\nypYtW6iqqsLMKCsrY8mSXbJ0F13mJ0ZsaoIRI+DKK2HmzCJUzGWGT4zo+ro9Jp/HnqCkBLZuDb/1\ncM45l5+iBg9JpwN3ETIJLjCzOW2scx4wCzDgRTO7IJZfCtwcV5ttZm1fw/VYXYu5d+eyKwv5PNau\nXcvFF1/8ibL+/ftTV1fX7X33VUUbtpLUD9gIfBmoJ+Q0rzGz9Yl1RgKLgXFmtkPSEDPbLukAYDVw\nLCGoPAccY2Y72nu+nkhD61x3+LCV6+v2lHwe1cCrZrbZzD4EfgOclbPOlcDdrUHBzLbH8tOAWjN7\nMz5WS8hz7lyflpZ7iC59evq9WczgMRR4PbFcH8uSDgMOk/S0pGfiMFe+2yLpKkmrJa1uaGjowao7\nV7jS0lIaGxs9gLg+x8xobGyktLS0x/ZZzHsebd1FyP1U7Q2MBE4GyoGnJB2V57aY2XxgPoRhq+5U\n1rnuKi8vp76+Hj+RcX1RaWkp5eXlPba/YgaPeuDgxHI58K821nnGzJqA1yRtIASTekJASW67smg1\nda4HlJSU7PYpIpzrLcUctloFjJR0qKR9gMnA0px1lgCnAEgaTBjG2gwsAyZI2l/S/sCEWOacc64P\nKNqVh5k1S7qGcNDvB9xnZi9LupUwi+NSPg4S64H/AdPNrBFA0vcJAQjgVjN7s1h1dc45V5jM/8Lc\nOeeyrKtf1U1N8JDUAPyjgE0GA8XJr7nnyPpr4O339nv74RAzKyt049QEj0JJWt2VaJsmWX8NvP3e\nfm9/19vvU7I755wrmAcP55xzBcty8Jjf2xXoA7L+Gnj7s83b3w2ZvefhnHOu67J85eGcc66LMhk8\nJJ0uaYOkVyXd1Nv1KTZJB0taIekVSS9Lui6WHyCpVtKm+P/+vV3XYpLUT9IaSb+Py4dKqovt/22c\nCSGVJA2S9LCkv8X3wfFZ6n9JN8T3/jpJD0kqTXv/S7pP0nZJ6xJlbfa5gp/GY+JLkqo623/mgkfM\nM3I3cAZwJFAj6cjerVXRNQPfMrMjgOOAq2ObbwKeMLORwBNxOc2uA15JLN8G/CS2fwdwRa/Uave4\nC/iTmR0OVBJeh0z0v6ShwDeBY83sKMKMF5NJf/8/wK6pLNrr8zMI8wqOBK4C5nW288wFD/LLM5Iq\nZrbNzJ6Pf79LOHAMJbS7NUPjQuDs3qlh8UkqByYCC+KygHHAw3GV1LZf0qeBk4B7AczsQzN7iwz1\nP2Eqpn0l7Q0MALaR8v43s78AudM6tdfnZwGLLHgGGCTpcx3tP4vBI69cIWklaThwNFAHHGhm2yAE\nGGBI79Ws6O4Evg20xOXPAm+ZWXNcTvP7oAJoAO6Pw3YLJO1HRvrfzP4J/AjYSggabxOyk2al/5Pa\n6/OCj4tZDB555QpJI0kDgUeA683snd6uz+4i6Uxgu5k9lyxuY9W0vg/2BqqAeWZ2NLCTlA5RtSWO\n658FHAocBOxHGKbJldb+z0fBn4csBo988oykjqQSQuB40MwejcVvtF6axv+3t7f9Hu4E4KuSthCG\nKccRrkQGxWEMSPf7oB6oN7O6uPwwIZhkpf9PBV4zs4aYO+hR4Itkp/+T2uvzgo+LWQwe+eQZSZU4\nvn8v8IqZ/Tjx0FLg0vj3pcBju7tuu4OZzTCzcjMbTujvP5vZhcAK4Jy4Wprb/2/gdUlfiEXjgfVk\npP8Jw1XHSRoQPwut7c9E/+dor8+XApfEb10dB7zdOrzVnkz+SFDSVwhnnq15Rn7Qy1UqKkknAk8B\na/l4zP87hPsei4FhhA/YuWnPmyLpZOBGMztTUgXhSuQAYA1wkZl90Jv1KxZJowlfFtiHkHBtCuHk\nMRP9L+kW4HzCNw/XAFMJY/qp7X9JDxEysg4G3gC+R0jAt0ufx6D6c8K3s/4DTDGzDnNcZDJ4OOec\n654sDls555zrJg8ezjnnCubBwznnXME8eDjnnCuYBw/nnHMF8+DhUkGSSbojsXyjpFk9tO8HJJ3T\n+Zrdfp5z44y3K3LKh0u6oNjP71whPHi4tPgAmCRpcG9XJCnO4pyvK4BvmNkpOeXDgTaDR+IX0s7t\nVh48XFo0E9Jq3pD7QO6Vg6T34v8nS3pS0mJJGyXNkXShpGclrZU0IrGbUyU9Fdc7M27fT9JcSati\nDoSvJfa7QtKvCT/MzK1PTdz/Okm3xbLvAicCv5Q0N2eTOcCXJL0Q81JcJul3kh4HlsftpyfqcUvi\nuS6K7XlB0j2xzv3ia7Iu1mOX18y5zvhZi0uTu4GXJN1ewDaVwBGEqas3AwvMrFohYda1wPVxveHA\nWGAEsELS54FLCNM4jJHUH3ha0vK4fjVwlJm9lnwySQcR8kgcQ8ghsVzS2WZ2q6RxhF+/5/6y96ZY\n3hq0LgOOB0bFXwdPIORhqCZMcLdU0kmEmXTPB04wsyZJvwAuBF4GhsbcFkgaVMDr5RzgwcOliJm9\nI2kRIfHP+3lutqp1Dh9JfyeeyROuGJLDR4vNrAXYJGkzcDgwARiVuKr5DOEg/iHwbG7giMYAK82s\nIT7ng4RcG0vyrG+r2sRUIhPivzVxeWCsxyhCkFoVZp9gX8JEeI8DFZJ+Bvwh0Wbn8ubBw6XNncDz\nwP2JsmbiEG2cwyeZbjQ5l1FLYrmFT34+cufxMcJZ/rVmtiz5QJw/a2c79Wtr6uuuSO5fwA/N7J6c\nelwLLDSzGbtUQqoETgOuBs4DLu+hermM8HseLlXi2fhiPplSdAvhDBxCXoeSLuz6XEl7xfsgFcAG\nYBkwLU53j6TDFJIsdaQOGCtpcLyZXgM82ck27wKf6uDxZcDlCvlakDRU0hBCmtFz4t+t+asPiV8q\n2MvMHgFmEqZnd64gfuXh0ugO4JrE8q+AxyQ9SzigtndV0JENhIP8gcDXzey/khYQ7oU8H69oGugk\nlamZbZM0gzAduIA/mllnU4G/BDRLepGQl3pHzj6XSzoC+GscnnqPMEPsekk3E+6r7AU0Ea403idk\nFWw9edzlysS5zvisus455wrmw1bOOecK5sHDOedcwTx4OOecK5gHD+eccwXz4OGcc65gHjycc84V\nzIOHc865gnnwcM45V7D/A84rA34C9WPNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x296d206d278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 算法测试与可视化\n",
    "num_trees = np.arange(1, 101, 5)\n",
    "np.random.seed(0)\n",
    "plt.figure()\n",
    "\n",
    "# bagging算法\n",
    "oob_score = []\n",
    "train_score = []\n",
    "with tqdm(num_trees) as pbar:\n",
    "    for n_tree in pbar:\n",
    "        rf = RandomForest(n_trees=n_tree, max_features=None)\n",
    "        rf.fit(X, y)\n",
    "        train_score.append(rf.score(X, y))\n",
    "        oob_score.append(rf.oob_score)\n",
    "        pbar.set_postfix({\n",
    "            'n_tree': n_tree, \n",
    "            'train_score': train_score[-1], \n",
    "            'oob_score': oob_score[-1]\n",
    "        })\n",
    "plt.plot(num_trees, train_score, color='blue', label='bagging_train_score')\n",
    "plt.plot(num_trees, oob_score, color='blue', linestyle='-.', label='bagging_oob_score')\n",
    "\n",
    "# 随机森林算法\n",
    "oob_score = []\n",
    "train_score = []\n",
    "with tqdm(num_trees) as pbar:\n",
    "    for n_tree in pbar:\n",
    "        rf = RandomForest(n_trees=n_tree, max_features='sqrt')\n",
    "        rf.fit(X, y)\n",
    "        train_score.append(rf.score(X, y))\n",
    "        oob_score.append(rf.oob_score)\n",
    "        pbar.set_postfix({\n",
    "            'n_tree': n_tree, \n",
    "            'train_score': train_score[-1], \n",
    "            'oob_score': oob_score[-1]\n",
    "        })\n",
    "plt.plot(num_trees, train_score, color='red', linestyle='--', label='random_forest_train_score')\n",
    "plt.plot(num_trees, oob_score, color='red', linestyle=':', label='random_forest_oob_score')\n",
    "\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Number of trees')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cell_id": "459eade73df14d3d8699008675468ed5",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 67
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 1634,
    "execution_start": 1672344895308,
    "id": "54C4CC5A64F54A7184C884025AE61640",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "source_hash": "13326553",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bagging： 0.884\n",
      "随机森林： 0.897\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier, RandomForestClassifier\n",
    "\n",
    "bc = BaggingClassifier(n_estimators=100, oob_score=True, random_state=0)\n",
    "bc.fit(X, y)\n",
    "print('bagging：', bc.oob_score_)\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=100, max_features='sqrt', oob_score=True, random_state=0)\n",
    "rfc.fit(X, y)\n",
    "print('随机森林：', rfc.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cell_id": "88173194e475444188774c689c9694bc",
    "collapsed": true,
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 91
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 4,
    "execution_start": 1672344906149,
    "id": "6499FABC7DFD4BB08BCA4D31BDB1272E",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "source_hash": "e7af5132",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "# 堆垛分类器，继承sklearn中的集成分类器基类EnsembleClassifier\n",
    "class StackingClassifier():\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        classifiers, # 基础分类器\n",
    "        meta_classifier, # 元分类器\n",
    "        concat_feature=False, # 是否将原始数据拼接在新数据上\n",
    "        kfold=5 # K折交叉验证\n",
    "    ):\n",
    "        self.classifiers = classifiers\n",
    "        self.meta_classifier = meta_classifier\n",
    "        self.concat_feature = concat_feature\n",
    "        self.kf = KFold(n_splits=kfold)\n",
    "        # 为了在测试时计算平均，我们需要保留每个分类器\n",
    "        self.k_fold_classifiers = []\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        # 用X和y训练基础分类器和元分类器\n",
    "        n_samples, n_features = X.shape\n",
    "        self.n_classes = np.unique(y).shape[0]\n",
    "        \n",
    "        if self.concat_feature:\n",
    "            features = X\n",
    "        else:\n",
    "            features = np.zeros((n_samples, 0))\n",
    "        for classifier in self.classifiers:\n",
    "            self.k_fold_classifiers.append([])\n",
    "            # 训练每个基础分类器\n",
    "            predict_proba = np.zeros((n_samples, self.n_classes))\n",
    "            for train_idx, test_idx in self.kf.split(X):\n",
    "                # 交叉验证\n",
    "                clf = clone(classifier)\n",
    "                clf.fit(X[train_idx], y[train_idx])\n",
    "                predict_proba[test_idx] = clf.predict_proba(X[test_idx])\n",
    "                self.k_fold_classifiers[-1].append(clf)\n",
    "            features = np.concatenate([features, predict_proba], axis=-1)\n",
    "        # 训练元分类器\n",
    "        self.meta_classifier.fit(features, y)\n",
    "    \n",
    "    def _get_features(self, X):\n",
    "        # 计算输入X的特征\n",
    "        if self.concat_feature:\n",
    "            features = X\n",
    "        else:\n",
    "            features = np.zeros((X.shape[0], 0))\n",
    "        for k_classifiers in self.k_fold_classifiers:\n",
    "            k_feat = np.mean([clf.predict_proba(X) for clf in k_classifiers], axis=0)\n",
    "            features = np.concatenate([features, k_feat], axis=-1)\n",
    "        return features\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return self.meta_classifier.predict(self._get_features(X))\n",
    "        \n",
    "    def score(self, X, y):\n",
    "        return self.meta_classifier.score(self._get_features(X), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cell_id": "5b6f59428c12428c91270439859e1ddd",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 103
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 1150,
    "execution_start": 1672344909280,
    "id": "53A5CC533DF74E7181CA011806310F4C",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "source_hash": "e55f08ba",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "随机森林： 0.895\n",
      "KNN： 0.9\n",
      "逻辑斯谛回归： 0.855\n",
      "Stacking分类器： 0.91\n",
      "带原始特征的Stacking分类器： 0.905\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "from sklearn.neighbors import KNeighborsClassifier as KNC\n",
    "\n",
    "# 划分训练集和测试集\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
    "\n",
    "# 基础分类器\n",
    "rf = RFC(n_estimators=10, max_features='sqrt', random_state=0).fit(X_train, y_train)\n",
    "knc = KNC().fit(X_train, y_train)\n",
    "# multi_class='ovr'表示二分类问题\n",
    "lr = LR(solver='liblinear', multi_class='ovr', random_state=0).fit(X_train, y_train)\n",
    "print('随机森林：', rf.score(X_test, y_test))\n",
    "print('KNN：', knc.score(X_test, y_test))\n",
    "print('逻辑斯谛回归：', lr.score(X_test, y_test))\n",
    "# 元分类器\n",
    "meta_lr = LR(solver='liblinear', multi_class='ovr', random_state=0)\n",
    "\n",
    "sc = StackingClassifier([rf, knc, lr], meta_lr, concat_feature=False)\n",
    "sc.fit(X_train, y_train)\n",
    "print('Stacking分类器：', sc.score(X_test, y_test))\n",
    "\n",
    "# 带原始特征的stacking分类器\n",
    "sc_concat = StackingClassifier([rf, knc, lr], meta_lr, concat_feature=True)\n",
    "sc_concat.fit(X_train, y_train)\n",
    "print('带原始特征的Stacking分类器：', sc_concat.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cell_id": "7ef6eb1f941c42c7919ef5dc3781aa25",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 157
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 10837,
    "execution_start": 1672344981631,
    "source_hash": "d405497b",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 20/20 [00:04<00:00,  4.32it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xdc1fX+wPHXBxQQRdwLB2au1ERF1LTSW2llZjcrrW5l\nZd5MG7br/iqz7rVhy7RMzWw4Uss0R5YrczIU3OYWnAwFWTLO+/fHFw6ggAc5xwP6fj4e34fnu98c\n8LzP5/tZRkRQSimlyhoPdweglFJKFUYTlFJKqTJJE5RSSqkySROUUkqpMkkTlFJKqTJJE5RSSqky\nSROUUkqpMkkTlFJKqTJJE5RSSqkyqYK7A3CWWrVqSWBgoLvDUEopVYiIiIg4EaldknMumwQVGBhI\neHi4u8NQSilVCGPMoZKeo4/4lFJKlUmaoJRSSpVJmqCUUkqVSZqglFJKlUmaoJRSSpVJmqCUUkqV\nSZqglFJKlUmaoJRSqoxKPpGMLdvm7jDcRhOUUkq5Uf4EtOy1ZSwcttC+PqPvDGbdOcu+fmj1Ic4c\nPXNJ43MnlyYoY8ytxpjdxpi9xphXC9nfxBiz3BizxRizyhjTMN++R4wxe3KWR1wZp1JKXQq2bBun\n9p+yry99fikT20/M259lw5aVl7C6Pd+NTv/uBIDYhJn9ZrL63dX2/Rs+3cCJLScuQeTu4bIEZYzx\nBCYAtwHXAPcbY64557CxwHcici0wGhiTc24N4C2gCxACvGWMqe6qWJVSyhVSTqaw8+ediAgAK/5v\nBRNaTyA7IxuAgJAAWvZvad/f+8Pe3Dn5Tvv57R5oR8s7W9rXH/rjIUKeDrGuHZvC0pFLObDiAAAZ\nyRnMe2geMRtjLsnPdim4sgQVAuwVkf0ikgHMAvqfc8w1wPKc1yvz7e8D/CEiCSJyCvgDuNWFsSql\nrmBisxLEye0nmX3PbI6EHgEg/u94Zg+YzbFNxwCI3RHL7AGz7aWWE1tOMHvAbGJ3xgJwNOIoswfM\nJn5PPAC7F+xm9oDZJOxNAKDNfW3oN7mf/X5tB7Xlpv/ehDHmgjEaD0NASAC1W1vjrVauXZlXTr1C\n0OAgAE4fOs3+ZftJi0+zfpZtJ5nceTJxu+Ks2MKPFoglen00swfM5vSh0wAc+usQswfMJulIEgAH\nVh5g9oDZpJxMubg31QlcOVhsABCdbz0Gq0SUXxQwAPgM+CfgZ4ypWcS5AefewBgzFBgK0LhxY6cF\nrpS6fKUnpmPLsuFb05fEw4lM6TqFWz64hWv/dS3eVb2J3R5LRnIGAFnpWcT/HU9GirWemZZJ/N/x\nZKZmApCRkkH83/FkpWVZ+1Ot/Vnp1nrLO1tS99q6VGtSDYD6HepTv0N9p/0sPtV87K/rtKnD80ef\nByv3kZmaiZeflz0We6xnc9aTrfXss1ZpLuNMznpO6e5s4llrPTPbafGWlMktWjr9wsbcC/QRkSE5\n6w8BISLydL5jGgDjgabAaqxk1QYr6XiLyLs5x70BpIrIR0XdLzg4WHQ0c6VUfiLC0bCjeHp5Ui+o\nHplpmbxX9T16vNaDXqN7Ycu28euQX2k/uD2BNwa6O9zLmjEmQkSCS3KOK0tQMUCjfOsNgaP5DxCR\no8DdAMaYKsAAEUk0xsQAPc85d5ULY1VKXSZ2zN2BLctG20FtMcYw5945NOzWkHtm3UPFShW57fPb\naNC5AQAenh70/+bcmgdVVriyDioMaG6MaWqM8QIGAQvyH2CMqWWMyY3hNWBqzuulQG9jTPWcxhG9\nc7apS2BH7A6GLxpOjfdr0H9Wf6IToy98klJulFunAxDxVQShn4fa1+/58R56j+1tXw9+MpgGnRpc\n0vjKk4S0BJbuXco7f75Dv5n92B23222xuKwEJSJZxpgRWInFE5gqItuNMaOBcBFZgFVKGmOMEaxH\nfMNzzk0wxryDleQARotIgqtiVZBly+LX3b8yPmw8Kw6swNvTm9ua38bv+36n46SOHHz2IJW9Krs7\nTKXOc3L7SX66/yfumXUPta+pzYCZA/Cpnlc307Brw2LOvrKlZaax+fhmQo+EEnoklLCjYexN2AuA\nwdC6dmtOppykZa2WF7iSa7h0Rl0RWQwsPmfbm/lezwXmFnHuVPJKVMpF0rPS+WT9J3wZ/iXRSdE0\n9m/MmJvG8HiHx6lduTYHTx8k9EioPTkdOHWAptWbujlqpfJ4+3nj6eVpr/z3reXr5ojKJpvY2BG7\ngypeVQisFkjk8Ug6T+5Mls163xpWbUhIQAhDOgwhJCCETg06UdW7qltjdlkjiUtNG0mUzLEzx6jv\nV59sWzYtx7ekafWmDO88nDta3EEFj8K/t8zfNZ8Bswew/OHl3Bh44yWOWKk8IsKB5QdoelNTjDGI\niENNtV3tbNZZok5EsTtuNy1rtaR93fZ4V/C+5HGICDFJMYQeCcWngg99W/TlbNZZ/Mb48Xy353nv\n5vdIy0zjndXv0CWgC50DOtPAz7WPPS+mkYQmqCvQK3+8wtebvybm+Rh8Kvhw5uwZ/Lz9LnheQloC\n4zaO4z/X/4eKnhWJT42npm/NSxCxUgXtnLeT2XfPZtCCQbTs557HT7lmbJ3Buuh1hB4JJfJ4JJm2\nTPu+ih4VCaoXxCPtH2F4yHAAlyTTU2mnCD8abj2qO2o9rjuefByAnoE9WfnISgAW/r2QtnXaElgt\n0Kn3d0RZa8WnyojoxGgmhk9kcNBgmtdszl2t7qKRfyN773VHkhNAjUo1GNVzFGD9h2j7ZVtuueoW\nPur9EbUr13ZV+Eqdp1X/Vtz13V206Nvikt97YvhE9iXs48PeHwLw4boP2Zuwl+AGwYzsOpKQgBBa\n1WrF7vjd9rqdpLNW59fkjGQaf9KYj/t8zOCgwaRnpROfGk9A1fO6eRYp25aNp4cnAK8te42fdv7E\nnoQ99v2tarWid7PehDQIISQghGvrXmvfd0eLO5zxFlwymqAuUyLCqoOrGB82nl92/QJA0+pNaV6z\nOd0adaNbo26lun6lipV4ouMTvLfmPRbvWczHfT7moWsfKhOPWdTlKWFvAkufX8pd0+6iUo1KtH+o\nvcvulXQ2Ka9EciSU7bHb2f7Udip4VGBn7E62ntxqP3bJg0uo7VvbnjRytanThrtb311gW2pmKoPa\nDqJ5jeYArItex03f3UQDvwaEBIQQ0iCELg27ENwgmKreVbGJjd1xu2lVqxXGGN5e9TaTN00memQ0\nxhgysjNoU6cNjwY9SkhACMENgvH38XfZ+3Kp6SM+J/k+6nvm757PseRjRR5TuWJlfn/odwDeWPEG\n+07tY8aAGQAMWziMLSe35B1s47xOAE38mxQ4vqJnRcbdNg6Ae2ffx76TR0hOhjPJkJh5kjTfvVTI\nqEndI0OoGz0Mn/QmzvuBc6RW3s6+a4Zypvo6/ONv4qodE6mUdrXT7+MMXl7Qvj2EhFhLs2ZwqfJp\nXByEhUFoqPXvqVMXPseVMrxOcjJgCil+UQQceIUqZzq6NyAHVEs6zLV/z2Vzq0GcqeL8+pKz3kc4\n0fAr4uvOJa3yLjDWZ6NPSnOqJIZw1a7PqZBVHUEwOOcP56x3NAl153GmaijJ/qGkV84pCYnBJ/Vq\nMr2Pk13hDB3+2kOltKtJqLWQM9XW0mjfKDzk0tVt+frCH3+U7hr6iM/FMrMz8fTwxMN4MH3LdD7b\n+BnrH1+Pp4cn66LXEXk8ksBqgUWWIipVrGR/7V3BG58KeU1hfTx9qL+nPonXJFL5cGVafNmC3cN3\nk9owNe+YfMdnnfXh2IkKvP669aG3ukYlMn2s1kuenlC1ckuaxvyHunED8bRVspKdCxo3+Uobam7/\niyP1JrEn8BWirmvHVdFv0fjIC3hIReffsBSSk2HSJPjsM2u9evW8ZBUSAp07Q926pb9Paips3mz9\nXnKX/futfcZA69bQwE3dcBKrhBLdYDwnav2IeGTgmeXHqdoL6RF2GK+sslmf6JMST3rlmmT4Niai\n9jOIZwWn/imn+PzNvib/R2ytnxFs1Dh9M/UPP4D/mRCqJgdTMauGdaBXzuKk5ATgSyOqxz0D1nB5\nZFZIIKlKOIl+oZypvAnvpJupeqYLVSvUpKIv+KbeAal3QKXir+tsvm5qGKklqCKICPtO7bMX8UOP\nhLLp2CY2DtlI+3rt+Xnnz0zdPJVv7/qWmr41CzwXvhgRkyNYOHQhj655lIq+FVk0bBEDfx6IXwM/\nkpIgPLzgB94RayxLKlSAa68t+EHbqpWVpC61o2eO8sySZ/hp50+0q9OOb/p/Q6cGnS59IMXIyoLt\n2/Pex7Aw2LoVbDkzHDRpUvC97NgRqlQp+nrZ2bBjR8Hfzdat1naARo0KXq9TJ/BzrMrPadKz0pm9\nfTbjQ8cTdjSMKl5VGNx+ME91for6fvVZe3gtfVv0BSDiaESZ+p1FfRfFgscX8NjaxwgIcbye5kJS\nMlJISEugkX8j9ibspeuUrjwa9ChPdX5Ku1G4iLbiK0WC2vr3GdYfXcXWhDCi4kLZEh9KYob1HKaS\npy9ta3bi2pohPNBiGE38mjkn6HyyM7I5vGQ7gXe249QpQ2ioEBZmCN0oZO7ax16aAYarry74gRcU\nBJUu8bepC5m/az5PL3maH+7+gRua3IBNbHiYsjs3ZkrK+SWeA9YMBnh4QJs2ee93+/Zw+HDecRER\n1vkA1apZpbD8JbL6hYwLmpyRjIfxwLei67+Wpmel02xcM46eOUqrWq0Y0XkED7V/qND+LUv3LuXW\n6bfy830/88/W/3R5bI44m3SWjZ9vpPvL3fGs6JxvXSJCy/Etuab2NfwyyKqfzcjOwMvTyynXv+xk\nZzvlG68mqFIkqCbNVtInfSa/3LmAWOrBkZC8JfYasDn/aWhdTtCTVfzMP8nk/P8ctWtDn8Y7uTpi\nNi3eGkTvp1tSs2w+hTlP/v/wg+YOIvFsIkseXALA/lP7aVi1YZn+QIiNzaszyl3i4/P2e3lBhw4F\nvyxcfbWV0Iqy6dgmJoROYMa2GWRmZ9K2TlurYjxnuab2NUX2QSuJddHrWLp3KW/3ehuAT9Z/wrV1\nr+UfTf9RbCOWs1lnmbxpMkM7DcXL04uDpw/SxL9JkeeICCeiTlAvqB5gzQZbpV4Vuj7b1aE4RYT1\nMetpU7sN/j7+zNw6k4/Wf0RX765c9cdV9P6wN63rti7VkwmwOqj+tvc35uyYw5R+U/D08GT29tkE\n+AXQvXH3Ul37snbiBEyeDF99ZVVAtWpVqstpHVQpvPFQY2Imt+CdVlH41nBCRYQDUqOSiJt2jFte\nTsKrfi379sqVrUdBjRsD0optP95N20EtLlmFvjPkTz7XN76e5Ixk+/qN024kNiWWDvU72JvChgSE\ncHWNq8tMK8DateH2260FQMQqVUVFWY/trr3WSlKO+n3f7/T5oQ++FX15+NqHqVulLqFHQpm7Yy6T\nN00GwLeiL3uf3kt9v/rsTdhLBY8KxSaI/M6cPYN3BW+8PL1YfWg1X4R/wTNdnqGmb01GdhvpUIze\nFbwZETICsFqxXff1dbSo2YKv7viq0KFuwr4I47dnf+OJsCeo36E+J7ecJP10un3/1B5Tad63Ode/\ndj1gTe/gVSXvTdseu53uU7vz2a2f8UyXZ6jsVRl/H3/CFoZRdX5Vbva7mdTGqQQ3CC6QyAP8Ahx6\nT06lneKbyG+YEDaB/af2U69KPQ6cPsDVNa7mvjb3OfSeXJHi4uC552D2bMjMhD59ICPDLaFoCSqf\n/B3oFo9YTLPezQrMZukMIkLCngRqtrCKQlnpWVTwcex7QkpsCr88/Au3fX4bNa6u4dS4LhURYc6O\nOfZ6vYhjEaRmWg1BqvtUp3NAZ7oEdKFv8750aXju9GHlh4gwatUo6lSuw/CQ4WRmZzJ502QeaPcA\n1XyqFTgut64z6ngU7938HsYYHvnlEZbuXcqxF45hjOHnnT/jW9GXzg06F+gcvTtuNxPCJjAtchoT\n75jIA+0eICUjBQ/jUaBRTknZxMbUzVN56Y+XSM1M5f+u/z9e6fFKgS8eZ8+cJXJaJCHDQzAeBROG\nLdvGgscX0OSGJnR4rANZ6VmM8R9D+sPppN2Txke3fMSR0COs817H7W1up4pXFdJOpVGpeiVsYiNy\neyRbs7YSdjSsQAdYD+NB0qtJVPaqzKqDqxARejXtVeDeUcejmBA2gR+2/EBaVho9GvdgROcR/LP1\nP8t0qd2t0tJgzx7rm1dmpvUsu3dveOopaOGcvmYXU4JCRC6LpVOnTuIsaafT5Mtrv5TV/1vttGvm\nWvfxOnnX512J3Rlb4nOPhB+Rjxt9LDEbY5wel7tkZmdK1PEomRwxWZ5Y8IQETQwSz7c95a2Vb4mI\nSGJ6otwz+x5Zd3idewN1gM1mk+0nt9vXb/3hVhkyf8hFXWvbiW2y6O9F9vUWn7cQRiGMQpp91kzu\n//JmueX9tsIoxOsdL/nXz/+SyGORpf4ZznXszDEZOGegMAq5ZsI1snTFUvn1379Kdma2Q+dn27Jl\n6d6l0n9Kf7n+puul8ZDG8q+f/yUnd5yUUYySTVM3iYhITGiMfFDrA0nYl1DoddIz02VjzEb5Pup7\n+7abv7tZOn7V0b7+39X/leunXi+MQiq9W0mGzB8im49tLsVPfwU4cEDklVdEatYUCQgQyciwtmc7\n9vstCaxBwkv0ue72xOKsxZkJSkQkKyNLsjKyRETk0F+HJHRCqNiybaW+bvLJZFnz/pqLvlZmeqb9\n9ckdJ0sdT1mUkpEip9JOiYj1Qd3006by257fRERk2b5l0u6LdvL4/Mflq/CvZPOxzZKRleHOcCX5\nbLJ8Ff6VtPuinXi+7SmHTx8WEXFqXEnpSbLywEp5//e3ZMDLgdJwJNL4OeTdPr5y/OR+p92nKAt3\nL5TGnzSWoP5B8ob/G7J/S/H3PJ12Wj7b8Jk9sdb5sI783/L/k+jEaBERSU9Klx0/75CkI0kiIrJn\nyR6Z9/A8yUh1/D07lXZKdsbuFBHry0GLz1vI1eOulrFrx0p8avxF/qRXiIgIkf79RTw8RDw9RQYM\nEFm5UsRW+s+4omiCcpFfn/xVPg38VM4mn72o8+P3xMvvL/8uNif+8g+tOSRve7wtUd9HOe2aZVnu\ne/fnwT/lth9uk5rv17SXKCq9W0m6f91dRv42UmZunSn7EvY59b0uyt74vfL8b89LtfeqCaOQ9l+2\nl8kRkyUlI8U1N/z7b5Hata0PlFdfFVm1SuTzz/P2v/KKyK+/imRlOf3WZ46dkTNnz8jI30aK76u+\nUn9sfVmyZ8l5x0UnRsuwhcOk8n8rC6OQrlO6yg9RP0h6ZrrTY1IllJgoEhdnvf7tN+tv6T//ETl8\n+JLcXhOUi9hsNkk6an3Ty87Klo3jN0pmWuYFzsqz/pP18n7N9yVhf+GPLy5GVkaW/DXmr4tOmuWd\nzWaTfQn7ZObWmTLyt5HS/evu4vOujz1pHTx1UERENkRvkIW7FzotYWXbsmXx34vl9um3ixllpMLo\nCjJo7iBZc2iN65Jies6He3a2yLBhIpGFPMpLSLAe0YBI06YiH34oEu+cUsTasWvl/Rrvy6mDVqk2\n7EiYdJjYQVYeWCki1mPaE8knRETkwKkD4vtfX3lk3iMSdiTMKfdXpbR9u8hTT4lUqSLy0kvWtuxs\nkbS0SxrGxSQobSRRQvt+38cPfX7gvp/uo/XdrYs9NrfSV0RIOZlClbrF9PgsheyMbJY+v5Qer/ag\nakP3zt+SX0ZyBgv/vZDGNzQm+N8lqxu9GJnZmWyP3c6mY5t4NOhRjDH86+d/sfLgSo48b/Vsvv+n\n+9kYs7HY6wRWC2TFIyvsxwPMHDATgJDJIYQdDaNelXo82elJhnYaSn2/Qjo7OUNWFnz6qTX0RXj4\nhYe5yMyEX36B8eNh9Wrw8YEFC+CWW0oVRsK+BDZN2cQ/3vkHHhWsdvT5+7Y9Pv9xjqccZ9EDiwAc\nHh2/zImMtJpVf/CB1ZR28mQYM+b84zZsgDp1rN9L7rAk+W3dap3/3//C11+fvz93WJHXX4dZswru\nq1zZOh/g2Wfh118L7q9Tx7o/wOOPw8qVBfc3bQrLl1uv778f1q+HQ4fA2xsGDoRnnrGaCLuBNjO/\nBJr1bmY1q+1kfShFr4umZsua+NYs2Oly7Ydr2fjZRoZGDKVK3SouS04AsTtiifouisY9GtN2UFuX\n3aekstKzOLzmMJ2HdwYgblccexbvoeMTHfH2c/44YhU9rakNguoF2bdNvGMih04fsq+3q9OOih7F\nD8FUr0q9Asfnd1PTm3ih2wuubxEWEQFPPGH1IL7jjryhKYpTsSLce6+1bNkCEydavYUBliyBxES4\n+26H2scnHk5k58876fpcV2o0q8HNY24usD83OdnERr0q9ejfqr99X7lKThkZMG+eldTXrLF6vQ8c\nCDfcAAEB0KPH+efkvn+NGxe+P7dTa9Omhe/PdfXV5+/3yRvOjJYtzx+0sVpeC1DatLG+lORXL+9v\nl3btrL+JNm3gscesvhPlTUmLXGV1ceUjvqJkpmfK2PpjZUa/GeftOx51XJY8t8Te0MLVkk8k21+n\nxLmoDsRBcbvj7I1A8jfqWPPBGnnH+x1JPmnFWpLHpFeM7GyR55+3Kq/r1ROZM8c5Fdf//Kf1+K9e\nPZG33hI5cqTYw5f/Z7mMqTpGEqMTS3/vsurECZH69a335aqrRD76yHpUqlwCfcR36UczPx51HE8v\nT2q3rk3i4UR2zN1Bt+dLN5VFacTviWdKlyn0HtubDo91uOT3j9sdx1dBX9HrnV5c9+J15+1PjE7E\nv5E1HcCs/rPwqOjBfXMvUafJ3L/1UnYGTtibQFpCmn1suPg98ZxNPEuDYGsE2LjdcWQkZ9CgU876\nrjgy0zKp38EqdcfuiCU7I9s+AsPJ7SeRbKHutdYjvBO3D4bKVag7+V2oVo0TW05gPA112tQB4Hhk\nzt/cNSX4RmyzwdKlVklhyRLrW/5rr8Ho0VYfmE2bAMg6m00Fb09sWTYSfepSvUsLa5TdqKjzr9mi\nhfWtPDERtm07f3/r1lCjBiQkwM6d5+9v2xb8/a1hO/7+GwIDrVKLq4jAunXW4IlPPGFtGznSegR6\n663FDwOiSk37QbnZ7y//LmOqjrGXENwhMy1TFg1fZK/QvtRsNpus/XCtQ+/Buo/XyYZxG+znhX0Z\nJmeOnXFdcDNmiPTrd1GtljJS8po/Lxi6QMbWG2tfn/fIPPmkySf29bn3z5VxzcfZ138c8KNMaDMh\nL4w7ZshXHb+yr3/fc4pMqfWKVZktIt/cMFW+ufEb+/4p3abId7d8Z1//qtNXMr3vdPv6ouGLJPyr\ncMd/mD17rFLarFnW+q5dIiDr6CZfMEzS8LZKFV/lxBgWZq2fu8zIeXKwcmXh+3/9NecNW1D4/pUr\nrf3Tp+dta9BA5K67RP73P5HYkvcVLFRqqsiUKSJBQdY96tcXOXtlNi5yJ7QE5d4p35NPJGPLtJWp\nhgq/Dv2VGlfXoPvLrhtzLCs9i2WvLuO6l66jasDF/eyxO2L5os0X9P2yL8FPBpP7d1nqoY+OHrXq\ncfr2tebaGDnS+qb83//C8OEODYJ5JOwIM26fwcB5A2ncozGxO2JJiU0h8MZAAE5uO0naqTSaXG/N\nt3Vi6wnOJp2lcffGgFXKzkzJpNF1jQA4tvkY2WezadglAKZO5dhz72NLzyBg6jvw0EMcDT8KYC+R\nHQk9gvE09hJZzMYYPL08qd+hPiLCtz2/pVH3Rtz0v5sQEb5s9yWd/t2JLk9bI3GkJ6bj45+vbuNc\nKSmwbh0HNieyZUUsfZ++igpeHlYJqGFDq4QUGnr+ee3aWXUeCQlWndm5goKsElZsrNUA4VydOlkl\nrGPHrDqz3bvzBj7csweOH7cahkybBqtW5Q16WJJxpubPt+pfEhKseEeMgAcftBojqEtKS1CqgKyM\nLJl972xZ8cYKl94ndmesjKk6xj4qwMWK2x0nZ89Y32y3z9kuEztMLF0dSESE1fS6Zk2RJKubgOzf\nL9Knj/VNOiREJKrofmS59Whnk8/KnIFznNsxevdukRtvtOK44QarFOMEZ5PPyryH58n2OVZpLCU2\nRUYxSsK+tJp8Z6ZnSvSGaHvdYPKJZNm9cLdT7u1Up/I9ARgzRqROnbxSlpeXyPXX59XNxcfnjXyQ\nnW318dmcM4LErl0i99wj8uefLu2Eqi4M7QelCpP7QRsTGiOLhi+S9CTndJpMjU+1v87fSMMuLk7k\nk09EvvuuxEOn7FqwS6bfPl2ys6zzDv11SE4dKMFjy3nzRHx9RRo1Oj8J2WzWY6XatUXmzi309I3j\nN8o3N35jv7/TvfqqSLVq1qMnFwwrkyslLkVW/2+1HI86LiIih9cdllGMkp3zrBEYfhn8i4zxHyNp\npy5tn5gSs9lEDh2yGo289JLIv/+dt+/660X8/UVuvlmkeXPrY23wYPfFqgqlCUoVa/0n6+XjRh9L\n2unSfxgd3XRUxviPkR0/7Th/Z0SEyKOPivj4WH9izZpZ9QAXyWazyfjW4+Xr7l87dsL774sYY5WQ\njh0r+rjExLxv1VOmiPzxh31X1A9R8uOAHyU90QnJ3GYT+esvkUGDrA9YEZGUlOJjc5G0U2my4+cd\n9paeqQmpciS8+BZ9Zd706SJPPinSoYOVrKZPz+vcrMoMTVDqgnIfodmybbLizRWSGHNxj9Ay0zNl\nwdAFcvrw6YI7fvjB+rOqXNka9WDr1rwmzSkpImPHXtSHx+nDp+XYZusDPSMlQ6Z0nSJ7luwp/OCX\nXxa57z7Hk2JWlmS0DZLF3CZbrh8mEhvrnFEhUlJEJk8Wad/eek/8/UW++KL011WqHLqYBKXtKq8w\nufPxnNx2knUfrGPf0n0On2vLsrFu7Doy0zKp4F2Bfl/1w98jGd58E+bMsQ7q29ca/eDIEfjiC6sp\ncQOrcp/58+HFF63K89WrSxS3fyN/e7PsM8fOAFDR1+pwmxqXSuy6PVZFO1i9/2fOdHyqYU9PPDes\n5WjDEOLX7IZWrTDff5/XLP1i3XKL1ZzZZrMmfTtyBIYNK901lbqSlDSjldVFS1Ald/rQaXtJYf+K\n/XJi64n/sT5IAAAgAElEQVRijz/450EZZUbJlulbrIFK77nHGrjUGJEXX3Tspr/9JhIYaJUonnii\nVB0jc2Nf9fRcGcVbktjwmhI1H046kiQLhy20j2eYlZFllfi6dbN+pm3bHA8mt3L+3ntFknPq4377\nTWT1aq2cV0ourgTl0qQB3ArsBvYCrxayvzGwEtgMbAFuz9keCKQBkTnLxAvdSxPUxbPZbPJl+y9l\ncsjkQh9t5R8N4/iW4yIPPGD96VSvblVY7y/hdA/JydZ5np7WCAelsWyZJFetL1v9uomsXSsiIive\nWCHrP1l/wVMPrj4o71Z6V/YvPyf+7GwrseRasiRvnpxznT4t8umneZXzdeta/YaUUgWUqQQFeAL7\ngKsALyAKuOacYyYBw3JeXwMclLwEta0k99MEVTopsSkSu8vqGJmZlikHVx8UEZEjYUfk04YfytF/\nvZTX9HfBAqtRQUoph1TatCmvefXx4yIHD5bs/EmTRCpUEGnTxpp4TaxkO/POmTJ/yHz7YXG74+yv\nT2w7IVE/5LXqS4m9wM+wfbv13+Taa0U2biy4LzraqmsDkeuuszquagdQpQp1MQnKlXVQIcBeEdkv\nIhnALKD/OccIkNuz0x846sJ4VDF8a/lSq2UtADaO28i0G6YR+90Sqr4yjOoxW/Ga+S2sXWsd3K+f\nNZKyr28xV3RAhw7WgJgAL79sDWr56aeODYxqs1kjd99yizV8TWAgYHXsHTR/EHd8eQdgDUs0vtV4\nwidanbj/evcvlr+6nKz0LPvPXaxrrrEGE42Ph65drTqljz6y9jVsCK+8YnVSXbvWGj3a0Q6kSqkL\nK2lGc3QB7gGm5Ft/CBh/zjH1ga1ADHAK6CR5JagUrEd/fwLXF3GPoUA4EN64cWMX5PwrU0ZKhmz7\nNlzE29saFubtt0WOHnXtTQ8eFLntNqs00qmTVboqTFKSVdoSsR4VZhY/4Gx6Urps/HyjfT6v5BPJ\nFzcUVWKiyIgRVt1U69YXvK9SqiDK0lBHxph7gT4iMiRn/SEgRESeznfM84ARkY+MMd2Ar4G2QEWg\niojEG2M6Ab8AbUQkqaj7lYWhji4ra9daUwH8+qs13cOlIAKzZ1tz1sTHw/ffW6WSXIcOWaW3ypWt\n+NwxuGdCgjXAqQNDJCml8pS1+aBigEb51hty/iO8x7EaUiAi640xPkAtETkJnM3ZHmGM2Qe0wCot\nqUuhc2drhOtmzS7dPY2x5uLp3Rveegv+8Q9re1qa1YS8f3/r9Zw57ht5ukYN99xXqSuQK/+XhwHN\njTFNjTFewCBgwTnHHAZuAjDGtAZ8gFhjTG1jjGfO9quA5sB+F8aqzuXlZdURVXXDwLfVq8O4cdZA\noTYb3HSTVZqrXNmaIbR370sfk1LqknNZghKRLGAEsBTYCcwWke3GmNHGmDtzDnsBeMIYEwXMBAbn\nPKu8AdiSs30u8KSIJLgqVlWIsWNh2TJ3R2E1mOjTB+66CzZutBotKKWuCDrdhjpfZiZUqWLVBX34\nobujUUpdBi6mDkqHOlLn27kTMjKsR3xKKeUmmqDU+XInl9MEpZRyI01Q6nyRkdZAqy1auDsSpdQV\nTBOUOt/Bg9a02trXRynlRq7sB6XKq59/hpQUd0ehlLrCaQlKFa5yZXdHoJS6wmmCUgUtX24NL3T8\nuLsjUUpd4TRBqYJWr7bGw3PHCBJKKZWPJihV0ObN1hQYpZ1KQymlSkkTlCooMlL7PymlygRNUCpP\nfDxER0NQkLsjUUopTVAqn5MnreQUXKLhspRSyiW0H5TK07q1VQellFJlgJaglFJKlUmaoFSe7t3h\n9dfdHYVSSgGaoFSu1FTYsAEqVnR3JEopBWiCUrm2brWmV9cWfEqpMkITlLLkzgGlCUopVUZoglKW\nzZvB3x8CA90diVJKAdrMXOXq2NFKUMa4OxKllAI0QalcQ4e6OwKllCpAH/EpqwVfcrK7o1BKqQI0\nQSmYP9+aXmPnTndHopRSdpqglNVAomJFuPpqd0eilFJ2mqCUlaDattVOukqpMkUT1JVOROeAUkqV\nSZqgrnRHjkBcnHbQVUqVOZqgrnS+vvD553DLLe6ORCmlCnBpgjLG3GqM2W2M2WuMebWQ/Y2NMSuN\nMZuNMVuMMbfn2/daznm7jTF9XBnnFa1GDRgxAlq2dHckSilVgMsSlDHGE5gA3AZcA9xvjLnmnMP+\nD5gtIh2AQcAXOedek7PeBrgV+CLnesrZ1q+HQ4fcHYVSSp3HlSWoEGCviOwXkQxgFtD/nGMEqJrz\n2h84mvO6PzBLRM6KyAFgb871lLM9+CC89JK7o1BKqfO4MkEFANH51mNytuU3CviXMSYGWAw8XYJz\nMcYMNcaEG2PCY2NjnRX3leP0aThwQFvwKaXKJFcmqMJGHZVz1u8HpolIQ+B24HtjjIeD5yIik0Qk\nWESCa9euXeqArzhRUda/mqCUUmWQKweLjQEa5VtvSN4jvFyPY9UxISLrjTE+QC0Hz1WltXmz9a82\nMVdKlUGuLEGFAc2NMU2NMV5YjR4WnHPMYeAmAGNMa8AHiM05bpAxxtsY0xRoDoS6MNYrU2Qk1Ktn\nLUopVca4rAQlIlnGmBHAUsATmCoi240xo4FwEVkAvABMNsaMxHqEN1hEBNhujJkN7ACygOEiku2q\nWK9Yo0fDkCHujkIppQplrHxQzAHG1AbeBQJE5I6cJuAhIjLtEsTnsODgYAkPD3d3GEoppQphjIkQ\nkeCSnOPII75pwJ/k1QntwSr5qPLswAFrBAlt/aiUKqMcSVB1RGQGYAMQkUxAH7eVdytXwjPPWE3N\nlVKqDHIkQaUYY2qQ08zbGNMZOOPSqJTrRUZClSrQrJm7I1FKqUI50kjiReBX4CpjzJ9YHWbvcWlU\nyvU2b4b27cFDxwtWSpVNxSaonE6znkAvoDVWB9odOUMXqfLKZrNKUIMHuzsSpZQqUrEJSkRsxpjP\nRKQrEHWJYlKudvgwpKRoB12lVJnmyCO+P4wx/UVkvsujUZdGYCAkJYEpbEQppZQqGxxJUCMAf2PM\nWSAN6zGfiEgNl0amXKtKFXdHoJRSxXIkQdVyeRTq0nrrLahTB4YPd3ckSilVpAs24coZYqgP8N+c\n5RYddqicmzQJQnVoQ6VU2XbBBGWM+S/wMrA/Z3nZGPOuqwNTLnL8uLVoAwmlVBnnyCO+fkCH3FKT\nMWYqsAlrunZV3kRGWv/qHFBKqTLO0V6aVfO99nNFIOoSyU1QWoJSSpVxjpSgPgA2GWOWY7Xg6wm8\n6cqglAtlZFjJqVo1d0eilFLFuuB0GwDGmACgC1aC2iAiR1wdWEnpdBtKKVV2uWS6DWPMnUCyiPws\nIj9hDR57x8UGqZRSSjnCkTqo0SKSmLsiIqeBd1wXknKZDRusx3u59VBKKVWGOZKgCjvGZVPFKxeK\niICoKKhZ092RKKXUBTmSoDYZYz4wxjQxxjQ2xnwIbHZ1YMoFIiOt5NSwobsjUUqpC3IkQY3IOW4+\n1rxQAE+5LCLlOps3W4/4dJBYpVQ5cMFHdSKSjDVpYe78UJVEJMXVgSkny8yEbdvg6afdHYlSSjnE\nkVZ83xljqhpjfIFtwAFjzPOuD0051ZkzcPfd0KuXuyNRSimHOPKIr52IJAF3Ab8DDYHBrgxKuUCN\nGjBjBtx+u7sjUUophziSoLyMMRWA/sAvOdO921wblnK61FR3R6CUUiXiSIKaAhwGqgN/GmMaA8ku\njUo5X9++cNtt7o5CKaUc5sh8UJ+ISAMR6S3WuEgxwD9cH5pyGhGriXmTJu6ORCmlHFbiDrciYgMy\nXBCLcpVDh+D0aZ1iQylVrjg63cZFMcbcaozZbYzZa4x5tZD9nxhjInOWv40xp/Pty863b4Er47zs\n6RQbSqly6IIlKGNMBRHJutC2Qs7zBCYAt2A9FgwzxiwQkR25x4jIyHzHPw3k/4qfJiL6ieoMkZHg\n4QHt2rk7EqWUcpgjj/hCgY4ObDtXCLBXRPYDGGNmYbUE3FHE8fcDbzkQjyqpnj3B2xt8fd0diVJK\nOazIBGWMqQPUByoZY9phzQUF1uy6jnzSBQDR+dZjsOaUKuxeTYCmwIp8m32MMeFAFvCeiPxSyHlD\ngaEAjRs3diCkK1TPntailFLlSHElqL7AY1gdcyeQl6DOAG84cO3CBnwranbEQcBcEcnOt62xiBw1\nxlwFrDDGbBWRfQUuJjIJmATWhIUOxHTlSUmB3buhbVvw8nJ3NEop5bAiE5SIfAN8Y4y5T0RmX8S1\nY4BG+dYbAkeLOHYQMPyc+x/N+Xe/MWYVVv3UvvNPVcXasAFuvhn++MP6VymlyglHWvHVMcZUBTDG\nTDTGhBpjbnLgvDCguTGmqTHGCysJndcazxjTEqsT8Pp826obY7xzXtcCulN03ZUqzuacmVG0BZ9S\nqpxxJEENFZEkY0xvrFLQMOCDC52U08pvBLAU2AnMFpHtxpjROdPI57ofmJXTCThXayDcGBMFrMSq\ng9IEdTEiI635n2rVcnckSilVIo604stNHLcB34hIRM60Gxc+UWQxsPicbW+esz6qkPPWAdom2hly\n54BSSqlyxpFEE2WMWQz0A5YYY6pQdGMHVZakpcGuXZqglFLlkiMlqEeBTlh9mlJz6oQed21Yyik8\nPWHpUmjU6MLHKqVUGePIYLHZwFVYdU8AlRw5T5UBXl5Wy72WLd0diVJKlZgjM+qOB3oB/8rZlAJM\ndGVQykmWLIFly9wdhVJKXRRHHvFdJyIdjTGbAUQkIafZuCrrRo+2hjjS/k9KqXLIkUd1mTmt9gTA\nGFMTnVG37MvOhi1bdIoNpVS5VWSCypnmHaxhjn4Cahtj3gbWAO9fgthUaezZY03zri34lFLlVHGP\n+EKBjiLynTEmArgZa3y9e0Vk2yWJTl283BEktASllCqniktQ9sFeRWQ7sN314SiniYqyWvG1bu3u\nSJRS6qIUl6BqG2OeL2qniHzsgniUs7z7LjzxBFSs6O5IlFLqohSXoDyBKhQ+bYYq6ypUgGbN3B2F\nUkpdtOIS1DERGX3JIlHOc/w4vPMOPPUUtGnj7miUUuqiFNfMXEtO5dXixfDFF9ZkhUopVU4Vl6Ac\nmfNJlUWTJ1uNIzp3dnckSil10YpMUCKScCkDUU6ydas1i+6QIWC0EKyUKr900NfLzeTJVvPyhx92\ndyRKKVUqmqAuN7VrW83LdQZdpVQ558hgsao8eeMNd0eglFJOoSWoy0loKNh0HF+l1OVBE9Tl4u+/\noUsXGD/e3ZEopZRTaIK6XEyZYo0ecd997o5EKaWcQhPU5SAjA6ZNg379oF49d0ejlFJOoQnqcjB/\nPsTGWq33lFLqMqEJ6nLw88/QuDH07u3uSJRSymm0mfnl4Pvv4cAB8PR0dyRKKeU0WoK6HFSoAM2b\nuzsKpZRyKk1Q5VlWFoSEwIwZ7o5EKaWczqUJyhhzqzFmtzFmrzHm1UL2f2KMicxZ/jbGnM637xFj\nzJ6c5RFXxlluLV4MYWFQubK7I1FKKadzWR2UMcYTmADcAsQAYcaYBSKyI/cYERmZ7/ingQ45r2sA\nbwHBgAAROeeeclW85dKkSVC/PvTt6+5IlFLK6VxZggoB9orIfhHJAGYB/Ys5/n5gZs7rPsAfIpKQ\nk5T+AG51YazlT0wMLFkCjz5q1UEppdRlxpUJKgCIzrcek7PtPMaYJkBTYEVJzjXGDDXGhBtjwmNj\nY50SdLkxdao17t7jj7s7EqWUcglXfvUubLY8KeLYQcBcEckuybkiMgmYBBAcHFzUtS9PvXpZExJe\ndZW7I1FKKZdwZYKKARrlW28IHC3i2EHA8HPO7XnOuaucGFv5d/311qKUUpcpVz7iCwOaG2OaGmO8\nsJLQgnMPMsa0BKoD6/NtXgr0NsZUN8ZUB3rnbFMA335rjV6ulFKXMZclKBHJAkZgJZadwGwR2W6M\nGW2MuTPfofcDs0RE8p2bALyDleTCgNE529Tx4zBkiDW1u1JKXcZc2vxLRBYDi8/Z9uY566OKOHcq\nMNVlwZVX06ZZHXSHDHF3JEoVKjMzk5iYGNLT090dinIDHx8fGjZsSMWKFUt9LW2fXJ7YbNa8Tzfc\nAC1bujsapQoVExODn58fgYGBGFNYeyd1uRIR4uPjiYmJoWnTpqW+ng51VJ6sWgX79um0GqpMS09P\np2bNmpqcrkDGGGrWrOm00rMmqPJk925o0AAGDHB3JEoVS5PTlcuZv3tNUOXJsGHWtBqVKrk7EqWU\ncjlNUOXF6ZxxdL283BuHUuWAp6cnQUFBtG/fno4dO7Ju3Tqn3yM8PJxnnnnG6ddVebSRRHkgAt26\nQY8e2rxcKQdUqlSJyMhIAJYuXcprr73Gn3/+6dR7BAcHExwc7NRrqoI0QZUHa9fCrl3w0kvujkSp\nEnnuOcjJE04TFASffur48UlJSVSvXh2A5ORk+vfvz6lTp8jMzOTdd9+lf39rDOt33nmH6dOn06hR\nI2rVqkWnTp148cUXCQsL4/HHH6dy5cr06NGDJUuWsG3bNlatWsXYsWNZuHAho0aN4vDhw+zfv5/D\nhw/z3HPP2UtXRV1XXZgmqPJg8mTw84OBA90diVLlQlpaGkFBQaSnp3Ps2DFWrLDGofbx8WHevHlU\nrVqVuLg4unbtyp133klERAQ//fQTmzdvJisri44dO9KpUycAHn30USZNmsR1113Hq6+eN62d3a5d\nu1i5ciVnzpyhZcuWDBs2jKioqCKvqy5ME1RZd/o0zJkDDz+sExOqcqckJR1nyv+Ib/369Tz88MNs\n27YNEeH1119n9erVeHh4cOTIEU6cOMGaNWvo378/lXIaIPXr1w+A06dPc+bMGa677joAHnjgARYu\nXFjoPfv27Yu3tzfe3t7UqVOn2Osqx2iCKutmzIC0NO37pNRF6tatG3FxccTGxrJ48WJiY2OJiIig\nYsWKBAYGkp6eTr6R1gooanthvL297a89PT3Jysoq0fnqfNqKr6x76CH48UfQxwJKXZRdu3aRnZ1N\nzZo1SUxMpE6dOlSsWJGVK1dy6NAhAHr06MGvv/5Keno6ycnJLFq0CIDq1avj5+fHhg0bAJg1a1aJ\n7l3UdZVjtARV1vn5wX33uTsKpcqV3DoosEpB3377LZ6enjz44IP069eP4OBggoKCaNWqFQCdO3fm\nzjvvpH379jRp0oTg4GD8/f0B+Prrr3niiSeoXLkyPXv2tG93RHHXVRdmLpciaHBwsISHh7s7DOca\nMwZq19aBYVW5snPnTlq3bu3uMEosOTmZKlWqkJqayg033MCkSZPo2LGjfTvAe++9x7Fjx/jss89K\nfd3LWWF/A8aYCBEpUbt8LUGVVcnJ8L//wb33aoJS6hIYOnQoO3bsID09nUceecSeRBYtWsSYMWPI\nysqiSZMmTJs2zSnXVRemCaqsmjXLSlLaOEKpS2LGjBmFbh84cCADS9HFo6jrqgvTRhLOtGmTNR3G\nli2lv9bkydCmDXTtWvprKaVUOaQlKGfJzrYex+3fDx07QkSEtX3CBKhVC0JCIDAQHBnpNyoKQkOt\nTiQ6KrRS6gqlCcpZliyxktOnn0JOpz5E4O23ITbWWq9VCzp3hgcftJaipKfDP/5hNTFXSqkrlCYo\nZxk3DgIC4KmnIHeqY2PgyBHYts0qEeUue/da+5OSrNJWcLBVwgoJsda7dIHly933syilVBmgdVDO\ncPasVVoaPjwvOeWqWBE6dIB//xu+/hq2boU337T2JSZa+9avhxdegOuvh0aNrIFhlVIXLXe6jbZt\n29KvXz9O505XU0oHDx6kbdu2TrlWfqNGjSIgIICgoCCCgoKKHfOvtCIjI1m8eLHLru9MmqCcwdsb\n/vgDHP2jyq1XatTIGmfv0CE4dgwWLLBKYCXsra6UKih3LL5t27ZRo0YNJkyY4O6QLmjkyJFERkYS\nGRnJe++95/B52dnZJbpPeUpQ+oivtJKSrKVhw9I1aKhXD/r1sxalLic9e56/7b77rC9jqalw++3n\n7x882Fri4uCeewruW7WqRLfv1q0bW3Ja1hY13cbBgwe57bbb6NGjB+vWrSMgIID58+dTqVIlIiIi\neOyxx/D19aVHjx7266anpzNs2DDCw8OpUKECH3/8Mb169WLatGn88ssvZGdns23bNl544QUyMjL4\n/vvv8fb2ZvHixdSoUcOh2JcvX86LL75IVlYWnTt35ssvv8Tb25vAwEAee+wxfv/9d0aMGEHnzp0Z\nPnw4sbGx+Pr6MnnyZFq1asWcOXN4++238fT0xN/fn2XLlvHmm2+SlpbGmjVreO2110rVhN7VtARV\nWpMmQdOmcPiwuyNRSp0jOzub5cuXc+eddwJ5021s2rSJlStX8sILL9gHdN2zZw/Dhw9n+/btVKtW\njZ9++gmwptsYN24c69evL3Dt3FLZ1q1bmTlzJo888gjp6ekAbNu2jRkzZhAaGsp//vMffH192bx5\nM926deO7774rNNZPPvnE/ohv6dKlpKenM3jwYH788Ue2bt1KVlYWX375pf14Hx8f1qxZw6BBgxg6\ndCiff/45ERERjB07lqeeegqA0aNHs3TpUqKioliwYAFeXl6MHj2agQMHEhkZWaaTE2gJqnSys2H8\neOjeHRo3dnc0SpVNxZV4fH2L31+rVolLTJA3Ft/Bgwfp1KkTt9xyC0CR020ANG3a1D5+X6dOnTh4\n8CCJiYmcPn2aG2+8EYCHHnqIJUuWALBmzRqefvppAFq1akWTJk34+++/AejVqxd+fn74+fnh7+9v\nn2ajXbt29tLcuUaOHFlgIsOoqCiaNm1KixYtAHjkkUeYMGECzz33HIA9uSQnJ7Nu3Truvfde+7ln\nz54FoHv37gwePJj77ruPu+++u8Tvo7tpCao0Fiyw6o9yZs5USpUNuXVQhw4dIiMjw17amT59un26\njcjISOrWrWsv9RQ1XYYp4tF9ceOY5r+Wh4eHfd3Dw4OsrCyHfoYLjZNaOWd+OJvNRrVq1ez1V5GR\nkezcuROAiRMn8u677xIdHU1QUBDx8fEO3bus0ARVGuPGWSWnnMcHSqmyxd/fn3HjxjF27FgyMzOL\nnG6jKNWqVcPf3581a9YAVoLLdcMNN9jX//77bw4fPkzLli2dFnurVq04ePAge3O6pXz//ff2klx+\nVatWpWnTpsyZMwewEltUVBQA+/bto0uXLowePZpatWoRHR2Nn58fZ86ccVqcrqQJ6mJFR8Nff1lN\nyyvok1KlyqoOHTrQvn17Zs2axYMPPkh4eDjBwcFMnz7dPt1Gcb755huGDx9Ot27d7DPjAjz11FNk\nZ2fTrl07Bg4cyLRp0wqUnErLx8eHb775hnvvvZd27drh4eHBk08+Weix06dP5+uvv6Z9+/a0adOG\n+fPnA/DSSy/Rrl072rZtyw033ED79u3p1asXO3bsICgoiB9//NFp8bqCTrdRGtHRULUq6PwuStmV\n1+k2lPM4a7oNl5agjDG3GmN2G2P2GmMK7SRkjLnPGLPDGLPdGDMj3/ZsY0xkzrLAlXGWWG5Sb9RI\nk5NSSrmIy55NGWM8gQnALUAMEGaMWSAiO/Id0xx4DeguIqeMMXXyXSJNRIJcFV+pfPABrFgB8+eD\nj4+7o1FKqcuSK0tQIcBeEdkvIhnALKD/Occ8AUwQkVMAInLShfE4R1aW1bQ8O1uTk1JKuZArE1QA\nEJ1vPSZnW34tgBbGmLXGmA3GmFvz7fMxxoTnbL+rsBsYY4bmHBMemztiuKvNmwcxMdq0XCmlXMyV\nzc8K6zxwbouMCkBzoCfQEPjLGNNWRE4DjUXkqDHmKmCFMWariOwrcDGRScAksBpJOPsHKNS4cXDV\nVdC37yW5nVJKXalcWYKKARrlW28IHC3kmPkikikiB4DdWAkLETma8+9+YBXQwYWxOmbTJlizBkaM\nAE9Pd0ejlFKXNVcmqDCguTGmqTHGCxgEnNsa7xegF4AxphbWI7/9xpjqxhjvfNu7Aztwt6ZNrQYS\njz7q7kiUUsXInW6jTZs2tG/fno8//hibzQZAeHg4z7jwEf2qVatYt27dRZ377LPPEhAQYI+1MIGB\ngcTFxRV7ncDAQNq1a0dQUBDt2rWz94tylk8//ZTU1FSnXrMwLktQIpIFjACWAjuB2SKy3Rgz2hiT\nO/TCUiDeGLMDWAm8JCLxQGsg3BgTlbP9vfyt/9ymenV46SWoVs3dkSilipE71NH27dv5448/WLx4\nMW+//TYAwcHBjBs3rlTXL264ootNUDabjXnz5tGoUSNWr15dmvAAWLlyJZGRkcydO9fpCflSJSiX\nDoEgIouBxedsezPfawGez1nyH7MOaOfK2Ers22+tESOKm6pdKXWentN6XvCYO1rcwYvXvWg/fnDQ\nYAYHDSYuNY57ZhecbmPV4FUlun+dOnWYNGkSnTt3ZtSoUfz555+MHTuWhQsX8ueff/Lss88CYIxh\n9erV+Pn58cEHH/D999/j4eHBbbfdxnvvvUfPnj257rrrWLt2LXfeeScPP/wwTz75JIdzZjL49NNP\nCQgIYOLEiXh6evLDDz/w+eef06pVq/OO6969+3lxrly5krZt2zJw4EBmzpxJz5xpSuLj47n//vuJ\njY0lJCSkwBh9d911F9HR0aSnp/Pss88ydOjQ866blJRE9erV7esff/wxU6dOBWDIkCH2wWcL256S\nksJ9991HTEwM2dnZvPHGG5w4cYKjR4/Sq1cvatWqxcqVK0v0+ygJHaPHEZmZ8NprcO21mqCUKoeu\nuuoqbDYbJ08W7MkyduxYJkyYQPfu3UlOTsbHx4clS5bwyy+/sHHjRnx9fUlISLAff/r0af78808A\nHnjgAUaOHEmPHj04fPgwffr0YefOnTz55JNUqVLFPjJ5Uceda+bMmdx///3079+f119/nczMTCpW\nrMjbb79Njx49ePPNN1m0aBGTJk2ynzN16lRq1KhBWloanTt3ZsCAAdSsWROwRlQXEfbv38/s2bMB\niCoO5loAABImSURBVIiI4JtvvmHjxo2ICF26dOHGG2/EZrMVun3//v00aNCARYsWAZCYmIi/vz8f\nf/wxK1eupFatWk78LZ1PE5Qj5s61Zrz9+mt3R6JUuVPSEk/+42v51irx+UUpbFi37t278/zzz/Pg\ngw9y991307BhQ5YtW8ajjz6Kr68vQIHJBfPPn7Rs2TJ27MireUhKSip0ENaijvPz87Nvy8jIYPHi\nxXzyySf4+fnRpUsXfv/9d/r27cvq1av5+eefAejbt2+B0tC4ceOYN28eANHR0ezZs8eeoHITyL59\n+7jpppvo2bMna9as4Z///Kd9JPS7776bv/76CxEpdPutt97Kiy++yCuvvMIdd9zB9ddf7+jb7RSa\noBwxbhw0bw59+rg7EqXURdi/fz+enp7UqVOnQOnl1VdfpW/fvixevJiuXbuybNmyYqfYyP0AB6vO\naP369QUGkC2MI8f99ttvJCYm0q6dVbORmpqKr68vfXO6sxQWz6pVq1i2bBnr16/H19eXnj172qcO\nya9Zs2bUrVuXHTt2FDmFR1HbW7RoQUREBIsXL+a1116jd+/evPnmm4Ue6wo6mvmFhIbChg3w9NPg\noW+XUuVNbGwsTz75JCNGjDjvg37fvn20a9eOV155heDgYHbt2kXv3r2ZOnWqvRFA/kd8+fXu3Zvx\n48fb1yMjIwHOm86iqOPymzlzJlOmTOHgwYMcPHiQAwcO8Pvvv5OamlpgWo8lS5Zw6tQpwHrcVr16\ndXx9fdm1axcbNmwoNM6TJ09y4MABmjT5//buPbqqOjvg+HcToplIBZElZWCGAEUEw00CgkEhCRYV\nrKgIDkFAMkNrUYY6WO2SrrYwuHQQmaJBShgVw4gThiLPKpOohTFLrYSQEMKrWIUCOhOKBoEIXMLu\nH+fczCXJzQNyuZec/VmLlZzfPY9ffvxyd87j7t2dtLQ01q1bR1VVFadOnWLt2rUMGzYsZPuXX35J\nfHw8kyZN4qmnnmL79u31/ozhYmdQjTl1Cm67DaZMiXRPjDFNFKio6/f7adu2LZMnT+bJJ5+ss95L\nL73E5s2biYmJoV+/fowaNYqrr76a0tJSbrnlFq666iruuecenn/++TrbZmdnM336dHw+H+fOnSMt\nLY2cnBxGjx7NuHHjWL9+PYsWLQq5XkBVVRX5+fksXbq0pu2aa65h6NChbNy4kdmzZzNhwgQGDBhA\neno6P3Srd48cOZKcnBx8Ph99+vQhNTX1gv4NHz6cmJgY/H4/8+bNo3PnznTu3JmsrCwGDx4MOA9D\npKQ4HzGtrz0/P5+nn36aNm3aEBsbW1Ny/tFHH2XUqFF06dIlrA9JWLkNY0yLsnIb5ooot3HF27oV\nvv020r0wxhhPsgAVypkzMHo0TJ0a6Z4YY4wnWYAKZdUqqKiAej74ZowxJvwsQNVHFV5+Gfr2hREj\nIt0bY4zxJHuKrz6ffALFxbBkCYT4PIQxxpjwsjOo+hQUOAlhJ0+OdE+MMcazLEDVZ84c2LsXgj41\nboy5cgTKbSQmJjJ69GgqKysvel8NlbcoKSlBRMjPzw+5/Zw5c1iwYEGDx5gzZw5du3YlOTmZm266\niccee6zBkhvNdSklQCLJAlRtZ844Xzt3jmw/jDEXLVBuo7y8nI4dO7J48eKwHCcvL4+hQ4eSl5d3\nyfuaOXMmpaWl7N69m507d9YkpW0JFqBag9OnoVcvWLQo0j0xptXIzcilNNdJ71PtryY3I5eyFWUA\n+Kv85GbkUv7bcgBOHz9NbkYue9Y4+fKq/q+K3Ixc9m3cB8DJP5xs9vGHDBnCkSNHapZffPFFBg0a\nhM/nY/bs2TXtDzzwAAMHDuTmm2++IGN4KKrK6tWryc3NpaCg4II8eM899xx9+vRhxIgR7Nu3r6b9\n1VdfZdCgQSQlJTF27Nh6ayqdPXuW06dP1ySFLS0tJTU1FZ/Px5gxY2pSHYVqz87Opl+/fvh8PjIz\nMzlw4AA5OTksXLiQ5ORkCgsLmzmCkWMBKlheHhw5AomJke6JMaYFVFdX88EHH3DffU6N1IKCAvbv\n38/WrVspLS2luLi4pjjgsmXLKC4uZtu2bWRnZ3Ps2LEG9/3RRx/Ro0cPevXqRUZGBu++65S+Ky4u\nZuXKlZSUlLBmzRqKiopqtnnwwQcpKipix44d9O3bl9eDKiQEAkiXLl248cYbSU5OBuCRRx7hhRde\noKysjP79+9cUXgzVPm/ePEpKSigrKyMnJ4eEhASmTZtWc4Z2uTOSXwoLUAGBR8sTE8EtFGaMuXRZ\nW7JIznLebGNiY8jakoVvkg+A2PhYsrZkkTje+aMwrn0cWVuy6PugkyYnvlM8WVuy6DO6DwDt/rxd\nk44ZyMV3/fXX8/XXX3PnnXcCToAqKCggJSWFAQMGsHfvXvbv3w84Zx5JSUmkpqbWlK5oSF5eHpmZ\nmQBkZmbWXOYrLCxkzJgxxMfHc+2119YER4Dy8nKGDRtG//79eeutt9i1a1fNa4EAUlFRwalTp1i5\nciXHjx+nsrKS9PR0AKZMmcKHH34Ysh3A5/MxceJEVqxYQdu2V/aD2hagAgoLYccOeOIJe7TcmCtc\n4B7UwYMHOXv2bM09KFVl1qxZlJaWUlpaymeffcbUqVMvKF2xY8cOUlJS6i1dEVBdXc3bb7/N3Llz\nSUhIYMaMGWzatKkmw3eoch1ZWVm88sor7Ny5k9mzZ9d7jNjYWEaOHHnRZd/feecdpk+fTnFxMQMH\nDmywPH20swAVkJ0NHTvCww9HuifGmBbSvn17srOzWbBgAX6/n7vvvptly5Zx8qRzL+vIkSNUVFQ0\nuXRFwPvvv09SUhKHDh3iwIEDHDx4kLFjx7Ju3TrS0tJYu3Yt3333HSdOnGDjxo012504cYIuXbrg\n9/trSmjUpqp8/PHH9OrVi/bt23PdddfV3Dd68803SU9PD9l+/vx5Dh06xPDhw5k/fz6VlZWcPHny\nspXHaGlX9vlfS5o/H/bsAbeKpjGmdUhJSSEpKYmVK1cyefJk9uzZw5AhQwBo164dK1asaLR0RW15\neXmMGTPmgraxY8eyZMkSNm3axPjx40lOTqZ79+4X3PN59tlnufXWW+nevTv9+/e/IGgsXLiQFStW\n4Pf78fl8PP744wAsX76cadOmUVVVRc+ePXnjjTdCtldXVzNp0iSOHz+OqjJz5kw6dOhQpwTIlXIf\nysptGGNalJXbMFZuwxhjTKtmAcoYY0xUsgBljGlxreXWgWm+lvy/twBljGlRcXFxHDt2zIKUB6kq\nx44dIy4urkX2Z0/xGWNaVLdu3Th8+DBHjx6NdFdMBMTFxdGtW7cW2ZcFKGNMi4qNjaVHjx6R7oZp\nBcJ6iU9ERorIPhH5TESeCbHOj0Rkt4jsEpHfBLVPEZH97r8p4eynMcaY6BO2MygRiQEWA3cCh4Ei\nEdmgqruD1ukNzAJuV9VvROQGt70jMBu4BVCg2N32m3D11xhjTHQJ5xnUYOAzVf1cVc8CK4H7a63z\nN8DiQOBR1Qq3/W7gPVX92n3tPWBkGPtqjDEmyoTzHlRX4FDQ8mHg1lrr3AggIh8BMcAcVf1diG27\n1j6AiDwKPOounhSRfbXXaUQnoP5Smd5lY1KXjUldNiZ12ZjUFTwm3Zu7cTgDVH3pfGs/d9oW6A1k\nAN2AQhFJbOK2qOqvgMYri4XqoMi25qbeaO1sTOqyManLxqQuG5O6LnVMwnmJ7zDwg6DlbsCX9ayz\nXlX9qvoFsA8nYDVlW2OMMa1YOANUEdBbRHqIyFVAJrCh1jrrgOEAItIJ55Lf50A+cJeIXCci1wF3\nuW3GGGM8ImyX+FT1nIj8FCewxADLVHWXiMwFtqnqBv4UiHYD1cDTqnoMQESexQlyAHNV9eswdPOi\nLw+2YjYmddmY1GVjUpeNSV2XNCatptyGMcaY1sVy8RljjIlKFqCMMcZEJc8GqKakYWrtROQHIrJZ\nRPa4qaaecNs7ish7bpqp99wHVTxDRGJEpERE/sNd7iEin7rj8Vv3oR9PEZEOIrJaRPa682WIl+eJ\niMx0f2fKRSRPROK8OE9EZJmIVIhIeVBbvfNCHNnue26ZiAxobP+eDFBBaZhGAf2ACSLSL7K9iohz\nwN+ral8gFZjujsMzwAeq2hv4wF32kieAPUHLLwAL3fH4BpgakV5F1svA71T1JiAJZ3w8OU9EpCvw\nd8AtqpqI8xBYJt6cJ7nUzfITal6MwvkYUW+cBAtLGtu5JwMUTUvD1Oqp6lequt39/gTOm05XnLFY\n7q62HHggMj28/ESkG/BXwGvusgB3AKvdVTw1HgAici2QBrwOoKpnVbUSD88TnCegvycibYF44Cs8\nOE9U9UOg9hPWoebF/cCv1fFfQAcR6dLQ/r0aoJqUSslLRCQBSAE+BTqr6lfgBDHghsj17LJ7CfgH\n4Ly7fD1Qqarn3GUvzpWewFHgDffS52sicg0enSeqegRYAPwvTmA6DhRj8yQg1Lxo9vuuVwNUk1Ip\neYWItAPeBn6mqt9Guj+RIiL3AhWqWhzcXM+qXpsrbYEBwBJVTQFO4ZHLefVx76ncD/QAvg9cg3P5\nqjavzZPGNPt3yasBylIpuUQkFic4vaWqa9zmPwZOvd2vFaG2b2VuB+4TkQM4l33vwDmj6uBeygFv\nzpXDwGFV/dRdXo0TsLw6T0YAX6jqUVX1A2uA27B5EhBqXjT7fderAaopaZhaPff+yuvAHlX916CX\nNgCBIpFTgPWXu2+RoKqzVLWbqibgzIn/VNWJwGZgnLuaZ8YjQFX/ABwSkT5u018Cu/HoPMG5tJcq\nIvHu71BgPDw9T4KEmhcbgEfcp/lSgeOBS4GheDaThIjcg/PXcSAN03MR7tJlJyJDgUJgJ3+65/KP\nOPehVgE/xPllfChMqaailohkAE+p6r0i0hPnjKojUAJMUtUzkezf5SYiyTgPjlyFky/zxzh/4Hpy\nnojIz4HxOE/ClgB/jXM/xVPzRETycKpRdAL+iFNodh31zAs3mL+C89RfFfBjVd3W4P69GqCMMcZE\nN69e4jPGGBPlLEAZY4yJShagjDHGRCULUMYYY6KSBShjjDFRyQKU8SwRURH5ZdDyUyIyp4X2nSsi\n4xpf85KP85CbXXxzrfYEEXk43Mc3JpwsQBkvOwM8KCKdIt2RYG62/aaaCjyuqsNrtScA9QaooGwH\nxkQ1C1DGy84BvwJm1n6h9hmQiJx0v2aIyO9FZJWI/LeIzBORiSKyVUR2ikivoN2MEJFCd7173e1j\nRORFESlya+L8bdB+N4vIb3A+OF27PxPc/ZeLyAtu278AQ4EcEXmx1ibzgGEiUurWLsoSkX8XkY1A\ngbv900H9+HnQsSa5P0+piCx1+xzjjkm52486Y2ZMS7O/pIzXLQbKRGR+M7ZJAvrilBn4HHhNVQeL\nU/BxBvAzd70EIB3oBWwWkb8AHsFJ8TJIRK4GPhKRAnf9wUCiqn4RfDAR+T5OraGBOHWGCkTkAVWd\nKyJ34GS8qP2J/Gfc9kBgzAKGAD73U/134dTlGYyTxHODiKThZC0fD9yuqn4R+TdgIrAL6OrWP0JE\nOjRjvIy5KBagjKep6rci8mucAnTfNXGzokAOMRH5H9wzEpwzn+BLbatU9TywX0Q+B24C7gJ8QWdn\n7XECxVlga+3g5BoEbFHVo+4x38Kpz7Suif0NeC8oFdFd7r8Sd7md2w8fTiAscjLT8D2cZJ8bgZ4i\nsgh4J+hnNiZsLEAZ4+Rk3A68EdR2DvcSuJtDLLh8d3B+tfNBy+e58Heqdh4xxTlbmaGq+cEvuLn/\nToXoX31lCi5G8P4F+IWqLq3VjxnAclWdVacTIknA3cB04EfAT1qoX8bUy+5BGc9zzypWcWGJ7gM4\nZxLg1P6JvYhdPyQibdz7Uj2BfUA+8Jhb5gQRuVGc4n8N+RRIF5FO7gMUE4DfN7LNCeDPGng9H/iJ\nOLXAEJGuInIDTonuce73iEhHEenuPkjSRlXfBv4Zp9yGMWFlZ1DGOH4J/DRo+VVgvYhsxXnTDnV2\n05B9OIGkMzBNVU+LyGs496a2u2dmR2mkNLiqfiUis3DKOQjwrqo2VsqhDDgnIjuAXJx7V8H7LBCR\nvsAn7qW8kzjZt3eLyD/h3OdqA/hxzpi+w6moG/ijts4ZljEtzbKZG2OMiUp2ic8YY0xUsgBljDEm\nKlmAMsYYE5UsQBljjIlKFqCMMcZEJQtQxhhjopIFKGOMMVHp/wEYquz/KMZBsgAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x296d3a05710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "# 初始化stump\n",
    "stump = DTC(max_depth=1, min_samples_leaf=1, random_state=0)\n",
    "\n",
    "# 弱分类器个数\n",
    "M = np.arange(1, 101, 5)\n",
    "bg_score = []\n",
    "rf_score = []\n",
    "dsc_ada_score = []\n",
    "real_ada_score = []\n",
    "plt.figure()\n",
    "\n",
    "with tqdm(M) as pbar:\n",
    "    for m in pbar:\n",
    "        # bagging算法\n",
    "        bc = BaggingClassifier(base_estimator=stump, n_estimators=m, random_state=0)\n",
    "        bc.fit(X_train, y_train)\n",
    "        bg_score.append(bc.score(X_test, y_test))\n",
    "        # 随机森林算法\n",
    "        rfc = RandomForestClassifier(n_estimators=m, max_depth=1, min_samples_leaf=1, random_state=0)\n",
    "        rfc.fit(X_train, y_train)\n",
    "        rf_score.append(rfc.score(X_test, y_test))\n",
    "        # 离散 AdaBoost，SAMME是分步加性模型（stepwise additive model）的缩写\n",
    "        dsc_adaboost = AdaBoostClassifier(base_estimator=stump, n_estimators=m, algorithm='SAMME', random_state=0)\n",
    "        dsc_adaboost.fit(X_train, y_train)\n",
    "        dsc_ada_score.append(dsc_adaboost.score(X_test, y_test))\n",
    "        # 实 AdaBoost，SAMME.R表示弱分类器输出实数\n",
    "        real_adaboost = AdaBoostClassifier(base_estimator=stump, n_estimators=m, algorithm='SAMME.R', random_state=0)\n",
    "        real_adaboost.fit(X_train, y_train)\n",
    "        real_ada_score.append(real_adaboost.score(X_test, y_test))\n",
    "\n",
    "# 绘图\n",
    "plt.plot(M, bg_score, color='blue', label='Bagging')\n",
    "plt.plot(M, rf_score, color='red', ls='--', label='Random Forest')\n",
    "plt.plot(M, dsc_ada_score, color='green', ls='-.', label='Discrete AdaBoost')\n",
    "plt.plot(M, real_ada_score, color='purple', ls=':', label='Real AdaBoost')\n",
    "plt.xlabel('Number of trees')\n",
    "plt.ylabel('Test score')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig('output_26_1.png')\n",
    "plt.savefig('output_26_1.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cell_id": "84372d45c1f448d2adf5f1342a31f268",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 187
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 3642,
    "execution_start": 1672346468352,
    "source_hash": "83a6e9d8",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xgboost in f:\\anaconda3\\lib\\site-packages (1.7.3)\n",
      "Requirement already satisfied: numpy in f:\\anaconda3\\lib\\site-packages (from xgboost) (1.21.5)\n",
      "Requirement already satisfied: scipy in f:\\anaconda3\\lib\\site-packages (from xgboost) (1.9.1)\n"
     ]
    }
   ],
   "source": [
    "# 安装并导入xgboost库\n",
    "!pip install xgboost\n",
    "import xgboost as xgb\n",
    "from sklearn.datasets import make_friedman1\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import BaggingRegressor, RandomForestRegressor, StackingRegressor, AdaBoostRegressor\n",
    "\n",
    "# 生成回归数据集\n",
    "reg_X, reg_y = make_friedman1(\n",
    "    n_samples=2000, # 样本数目\n",
    "    n_features=100, # 特征数目\n",
    "    noise=0.5, # 噪声的标准差\n",
    "    random_state=0 # 随机种子\n",
    ")\n",
    "\n",
    "# 划分训练集与测试集\n",
    "reg_X_train, reg_X_test, reg_y_train, reg_y_test = train_test_split(reg_X, reg_y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cell_id": "ed114fef61b4434eae7f106e53f8e79a",
    "deepnote_app_coordinates": {
     "h": 5,
     "w": 12,
     "x": 0,
     "y": 193
    },
    "deepnote_cell_type": "code",
    "deepnote_to_be_reexecuted": false,
    "execution_millis": 7710,
    "execution_start": 1672346475053,
    "source_hash": "4848b46c",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost：1.652\n",
      "KNN：4.471\n",
      "线性回归：2.525\n",
      "Bagging：4.042\n",
      "随机森林：4.514\n",
      "Stacking：2.231\n",
      "带输入特征的Stacking：2.288\n",
      "AdaBoost：3.116\n"
     ]
    }
   ],
   "source": [
    "def rmse(regressor):\n",
    "    # 计算regressor在测试集上的RMSE\n",
    "    y_pred = regressor.predict(reg_X_test)\n",
    "    return np.sqrt(np.mean((y_pred - reg_y_test) ** 2))\n",
    "\n",
    "# XGBoost回归树\n",
    "xgbr = xgb.XGBRegressor(\n",
    "    n_estimators=100, # 弱分类器数目\n",
    "    max_depth=1, # 决策树最大深度\n",
    "    learning_rate=0.5, # 学习率\n",
    "    gamma=0.0, # 对决策树叶结点数目的惩罚系数，当弱分类器为stump时不起作用\n",
    "    reg_lambda=0.1, # L2正则化系数\n",
    "    subsample=0.5, # 与随机森林类似，表示采样特征的比例\n",
    "    objective='reg:squarederror', # MSE损失函数\n",
    "    eval_metric='rmse', # 用RMSE作为评价指标\n",
    "    random_state=0 # 随机种子\n",
    ")\n",
    "\n",
    "xgbr.fit(reg_X_train, reg_y_train)\n",
    "print(f'XGBoost：{rmse(xgbr):.3f}')\n",
    "\n",
    "# KNN回归\n",
    "knnr = KNeighborsRegressor(n_neighbors=5).fit(reg_X_train, reg_y_train)\n",
    "print(f'KNN：{rmse(knnr):.3f}')\n",
    "\n",
    "# 线性回归\n",
    "lnr = LinearRegression().fit(reg_X_train, reg_y_train)\n",
    "print(f'线性回归：{rmse(lnr):.3f}')\n",
    "\n",
    "# bagging\n",
    "stump_reg = DecisionTreeRegressor(max_depth=1, min_samples_leaf=1, random_state=0)\n",
    "bcr = BaggingRegressor(base_estimator=stump_reg, n_estimators=100, random_state=0)\n",
    "bcr.fit(reg_X_train, reg_y_train)\n",
    "print(f'Bagging：{rmse(bcr):.3f}')\n",
    "\n",
    "# 随机森林\n",
    "rfr = RandomForestRegressor(n_estimators=100, max_depth=1, max_features='sqrt', random_state=0)\n",
    "rfr.fit(reg_X_train, reg_y_train)\n",
    "print(f'随机森林：{rmse(rfr):.3f}')\n",
    "\n",
    "# 堆垛，默认元学习器为带L2正则化约束的线性回归\n",
    "stkr = StackingRegressor(estimators=[\n",
    "    ('knn', knnr), \n",
    "    ('ln', lnr), \n",
    "    ('rf', rfr)\n",
    "])\n",
    "stkr.fit(reg_X_train, reg_y_train)\n",
    "print(f'Stacking：{rmse(stkr):.3f}')\n",
    "\n",
    "# 带有输入特征的堆垛\n",
    "stkr_pt = StackingRegressor(estimators=[\n",
    "    ('knn', knnr), \n",
    "    ('ln', lnr), \n",
    "    ('rf', rfr)\n",
    "], passthrough=True)\n",
    "stkr_pt.fit(reg_X_train, reg_y_train)\n",
    "print(f'带输入特征的Stacking：{rmse(stkr_pt):.3f}')\n",
    "\n",
    "# AdaBoost，回归型AdaBoost只有连续型，没有离散型\n",
    "abr = AdaBoostRegressor(base_estimator=stump_reg, n_estimators=100, learning_rate=1.5, loss='square', random_state=0)\n",
    "abr.fit(reg_X_train, reg_y_train)\n",
    "print(f'AdaBoost：{rmse(abr):.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "deepnote": {},
  "deepnote_app_layout": "article",
  "deepnote_execution_queue": [],
  "deepnote_notebook_id": "b15e3cce624d49d49495fc57a07bc4e4",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
